commit 5b21ea25ba1f9f24ab44debfac4598d1c8c7f896 Author: Navis Ryu Date: Wed Jul 20 15:37:31 2011 +0900 DPAL-41 column name backtracking (HIVE-2283) diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnTracker.java ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnTracker.java new file mode 100644 index 0000000..2d18bc3 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnTracker.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.hive.ql.plan.*; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ColumnTracker { + + public static String resolveString(Operator op, String input, int index) { + String[] resolved = resolve(op, new String[]{null, input}, index); + return toString(resolved); + } + + public static String resolveString(Operator op, ExprNodeDesc expr, int index) { + String[] resolved = resolve(op, expr, index); + return toString(resolved); + } + + private static String[] resolve(Operator op, String[] column, int index) { + index = index < 0 ? indexFor(op, column) : index; + return forward(op, column, index); + } + + private static String[] resolve(Operator op, ExprNodeDesc expr, int index) { + index = index < 0 ? indexFor(op, expr) : index; + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc column = (ExprNodeColumnDesc) expr; + return forward(op, new String[]{column.getTabAlias(), column.getColumn()}, index); + } + if (expr instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) expr; + + List args = new ArrayList(); + for (ExprNodeDesc child : func.getChildExprs()) { + args.add(resolveString(op, child, index)); + } + String[] array = args.toArray(new String[args.size()]); + return new String[]{null, func.getGenericUDF().getDisplayString(array)}; + } + return new String[]{null, expr.getExprString()}; + } + + @SuppressWarnings("unchecked") + private static int indexFor(Operator op, ExprNodeDesc expr) { + if (op instanceof CommonJoinOperator) { + JoinDesc join = ((CommonJoinOperator) op).getConf(); + return indexFor(expr, join.getKeys(), join.getExprs()); + } + if (op instanceof HashTableSinkOperator) { + HashTableSinkDesc hsync = ((HashTableSinkOperator) op).getConf(); + return indexFor(expr, hsync.getKeys(), hsync.getExprs()); + } + return -1; + } + + @SuppressWarnings("unchecked") + private static int indexFor(Operator op, String[] column) { + if (op instanceof CommonJoinOperator) { + JoinDesc join = ((CommonJoinOperator) op).getConf(); + return indexFor(column, join.getKeys(), join.getExprs()); + } + if (op instanceof HashTableSinkOperator) { + HashTableSinkDesc hsync = ((HashTableSinkOperator) op).getConf(); + return indexFor(column, hsync.getKeys(), hsync.getExprs()); + } + return -1; + } + + private static int indexFor(ExprNodeDesc expr, Map>... mappings) { + for (Map> mapping : mappings) { + for (Map.Entry> entry : mapping.entrySet()) { + for (ExprNodeDesc mexpr : entry.getValue()) { + if (mexpr.isSame(expr) && isSameTable(expr, mexpr)) { + return entry.getKey(); + } + } + } + } + throw new IllegalStateException("not found "); + } + + private static boolean isSameTable(ExprNodeDesc expr, ExprNodeDesc mexpr) { + if (expr instanceof ExprNodeColumnDesc && mexpr instanceof ExprNodeColumnDesc) { + String alias1 = ((ExprNodeColumnDesc) expr).getTabAlias(); + String alias2 = ((ExprNodeColumnDesc) mexpr).getTabAlias(); + return (alias1 == null && alias2 == null) || (alias1 != null && alias1.equals(alias2)); + } + return true; + } + + private static int indexFor(String[] column, Map>... mappings) { + for (Map> mapping : mappings) { + for (Map.Entry> entry : mapping.entrySet()) { + for (ExprNodeDesc mexpr : entry.getValue()) { + if (mexpr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc mcolumn = (ExprNodeColumnDesc) mexpr; + if ((column[0] == null || column[0].equals(mcolumn.getTabAlias())) && column[1].equals(mcolumn.getColumn())) { + return entry.getKey(); + } + } + } + } + } + throw new IllegalStateException("not found "); + } + + private static String toString(String[] column) { + return (column[0] == null ? "" : column[0] + ".") + column[1]; + } + + private static String[] forward(Operator op, String[] column, int index) { + Operator parent = parentFor(op, index); + if (parent != null) { + ExprNodeDesc expr = backtracking(parent, column); + return expr != null ? resolve(parent, expr, -1) : resolve(parent, column, -1); + } + return column; + } + + private static ExprNodeDesc backtracking(Operator op, String[] column) { + if (op instanceof SelectOperator && ((SelectOperator) op).getConf().isSelectStar()) { + // I think this is a bug from GroupBy handling + return null; + } + Map colMapper = op.getColumnExprMap(); + if (colMapper != null && colMapper.containsKey(column[1])) { + return colMapper.get(column[1]); + } + return null; + } + + private static Operator parentFor(Operator op, int index) { + if (op instanceof TableScanOperator && ((TableScanOperator) op).getSplitParent() != null) { + return ((TableScanOperator) op).getSplitParent(); + } + List> parents = new ArrayList>(); + if (index >= 0) { + Operator target = op.getParentOperators().get(index); + if (target == null || target instanceof HashTableDummyOperator) { + target = peerFor(op).getParentOperators().get(index); + } + return target; + } + if (op.getParentOperators() == null) { + return null; + } + if (op.getParentOperators().size() > 1) { + throw new IllegalStateException("ambiguous"); + } + return op.getParentOperators().get(0); + } + + private static Operator peerFor(Operator op) { + if (op instanceof MapJoinOperator) { + return ((MapJoinOperator) op).getPeer(); + } + if (op instanceof HashTableSinkOperator) { + return ((HashTableSinkOperator) op).getPeer(); + } + return null; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index fe95cb1..04f7244 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -23,13 +23,7 @@ import java.io.PrintStream; import java.io.Serializable; import java.lang.annotation.Annotation; import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; import java.util.Map.Entry; import org.apache.hadoop.fs.Path; @@ -37,6 +31,7 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ExplainWork; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.StringUtils; @@ -94,7 +89,7 @@ public class ExplainTask extends Task implements Serializable { } private void outputMap(Map mp, String header, PrintStream out, - boolean extended, int indent) throws Exception { + boolean extended, Operator current, int indent) throws Exception { boolean first_el = true; TreeMap tree = new TreeMap(); @@ -118,7 +113,7 @@ public class ExplainTask extends Task implements Serializable { out.println(); } else if (ent.getValue() instanceof Serializable) { out.println(); - outputPlan((Serializable) ent.getValue(), out, extended, indent + 2); + outputPlan((Serializable) ent.getValue(), out, extended, current, indent + 2); } else { out.println(); } @@ -126,7 +121,7 @@ public class ExplainTask extends Task implements Serializable { } private void outputList(List l, String header, PrintStream out, - boolean extended, int indent) throws Exception { + boolean extended, Operator current, int indent) throws Exception { boolean first_el = true; boolean nl = false; @@ -148,7 +143,7 @@ public class ExplainTask extends Task implements Serializable { if (first_el) { out.println(); } - outputPlan((Serializable) o, out, extended, indent + 2); + outputPlan((Serializable) o, out, extended, current, indent + 2); } first_el = false; @@ -174,7 +169,7 @@ public class ExplainTask extends Task implements Serializable { } private void outputPlan(Serializable work, PrintStream out, boolean extended, - int indent) throws Exception { + Operator current, int indent) throws Exception { // Check if work has an explain annotation Annotation note = work.getClass().getAnnotation(Explain.class); @@ -192,11 +187,11 @@ public class ExplainTask extends Task implements Serializable { if (work instanceof Operator) { Operator operator = (Operator) work; if (operator.getConf() != null) { - outputPlan(operator.getConf(), out, extended, indent); + outputPlan(operator.getConf(), out, extended, operator, indent); } if (operator.getChildOperators() != null) { for (Operator op : operator.getChildOperators()) { - outputPlan(op, out, extended, indent + 2); + outputPlan(op, out, extended, op, indent + 2); } } return; @@ -220,6 +215,9 @@ public class ExplainTask extends Task implements Serializable { if (val == null) { continue; } + if (extended && xpl_note.columnTrack()) { + val = trackColumns(current, val); + } String header = null; if (!xpl_note.displayName().equals("")) { @@ -239,7 +237,7 @@ public class ExplainTask extends Task implements Serializable { try { // Go through the map and print out the stuff Map mp = (Map) val; - outputMap(mp, header, out, extended, prop_indents + 2); + outputMap(mp, header, out, extended, current, prop_indents + 2); continue; } catch (ClassCastException ce) { // Ignore - all this means is that this is not a map @@ -248,7 +246,7 @@ public class ExplainTask extends Task implements Serializable { // Try this as a list try { List l = (List) val; - outputList(l, header, out, extended, prop_indents + 2); + outputList(l, header, out, extended, current, prop_indents + 2); continue; } catch (ClassCastException ce) { @@ -259,7 +257,7 @@ public class ExplainTask extends Task implements Serializable { try { Serializable s = (Serializable) val; out.println(header); - outputPlan(s, out, extended, prop_indents + 2); + outputPlan(s, out, extended, current, prop_indents + 2); continue; } catch (ClassCastException ce) { @@ -270,6 +268,37 @@ public class ExplainTask extends Task implements Serializable { } } + private Object trackColumns(Operator current, Object val) { + if (val instanceof List) { + List details = new ArrayList(); + for (Object column : (List)val) { + details.add(trackColumns(current, column)); + } + return details; + } + if (val instanceof Map) { + Mapresult = new HashMap(); + for (Map.Entry entry : ((Map)val).entrySet()) { + result.put(entry.getKey(), trackColumns(current, entry.getValue())); + } + return result; + } + return trackColumn(current, val); + } + + private String trackColumn(Operator current, Object val) { + if (val instanceof String) { + String column = (String) val; + String resolved = ColumnTracker.resolveString(current, column, -1); + return column.equals(resolved) ? column : column + "=" + resolved; + } + if (val instanceof ExprNodeDesc) { + ExprNodeDesc expr = (ExprNodeDesc) val; + return expr.getExprString() + "=" + ColumnTracker.resolveString(current, expr, -1); + } + return String.valueOf(val); + } + private void outputPlan(Task task, PrintStream out, boolean extended, HashSet> displayedSet, int indent) throws Exception { @@ -283,7 +312,7 @@ public class ExplainTask extends Task implements Serializable { out.printf("Stage: %s\n", task.getId()); // Start by getting the work part of the task and call the output plan for // the work - outputPlan(task.getWork(), out, extended, indent + 2); + outputPlan(task.getWork(), out, extended, null, indent + 2); out.println(); if (task instanceof ConditionalTask && ((ConditionalTask) task).getListTasks() != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java index 9cffc10..9eea29f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java @@ -118,6 +118,8 @@ public class HashTableSinkOperator extends TerminalOperator i private long hashTableScale; private boolean isAbort = false; + private transient Operator peer; + public static class HashTableSinkObjectCtx { ObjectInspector standardOI; SerDe serde; @@ -443,4 +445,12 @@ public class HashTableSinkOperator extends TerminalOperator i public OperatorType getType() { return OperatorType.HASHTABLESINK; } + + public Operator getPeer() { + return peer; + } + + public void setPeer(Operator peer) { + this.peer = peer; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index c7784c9..52284a5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -67,6 +67,8 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem transient boolean hashTblInitedOnce; private int bigTableAlias; + private transient Operator peer; + public MapJoinOperator() { } @@ -153,7 +155,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem hashTblInitedOnce = true; } } - + boolean localMode = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPJT).equals("local"); String baseDir = null; @@ -301,7 +303,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem public void closeOp(boolean abort) throws HiveException { if (mapJoinTables != null) { - for (HashMapWrapper hashTable : mapJoinTables.values()) { + for (HashMapWrapper hashTable : mapJoinTables.values()) { hashTable.close(); } } @@ -322,4 +324,12 @@ public class MapJoinOperator extends AbstractMapJoinOperator implem public OperatorType getType() { return OperatorType.MAPJOIN; } + + public Operator getPeer() { + return peer; + } + + public void setPeer(Operator peer) { + this.peer = peer; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java index 141df13..8f91658 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java @@ -60,6 +60,8 @@ public class TableScanOperator extends Operator implements private transient Stat currentStat; private transient Map stats; + private Operator splitParent; + public TableDesc getTableDesc() { return tableDesc; } @@ -262,4 +264,12 @@ public class TableScanOperator extends Operator implements } statsPublisher.closeConnection(); } + + public Operator getSplitParent() { + return splitParent; + } + + public void setSplitParent(Operator parent) { + this.splitParent = parent; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index ae33079..0fd04fc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -952,11 +952,12 @@ public final class GenMapRedUtils { // create a dummy tableScan operator on top of op // TableScanOperator is implicitly created here for each MapOperator RowResolver rowResolver = opProcCtx.getParseCtx().getOpParseCtx().get(parent).getRowResolver(); - Operator ts_op = putOpInsertMap(OperatorFactory + TableScanOperator ts_op = (TableScanOperator) putOpInsertMap(OperatorFactory .get(TableScanDesc.class, parent.getSchema()), rowResolver, parseCtx); childOpList = new ArrayList>(); childOpList.add(op); + ts_op.setSplitParent(parent); ts_op.setChildOperators(childOpList); op.getParentOperators().set(posn, ts_op); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java index 8f2fef4..6d26238 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java @@ -181,6 +181,8 @@ public final class LocalMapJoinProcFactory { for (Operator op : dummyOperators) { context.addDummyParentOp(op); } + hashTableSinkOp.setPeer(mapJoinOp); + mapJoinOp.setPeer(hashTableSinkOp); return null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dc9ab1b..309ece3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -4579,13 +4579,20 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { rightOps[pos] = input; } + Map> keyExprMap = new HashMap>(); + for (int pos = 0; pos < rightOps.length; pos++) { + ReduceSinkOperator rs = (ReduceSinkOperator) rightOps[pos]; + ReduceSinkDesc rsconf = rs.getConf(); + keyExprMap.put((byte) rsconf.getTag(), rsconf.getKeyCols()); + } + JoinCondDesc[] joinCondns = new JoinCondDesc[join.getJoinCond().length]; for (int i = 0; i < join.getJoinCond().length; i++) { JoinCond condn = join.getJoinCond()[i]; joinCondns[i] = new JoinCondDesc(condn); } - JoinDesc desc = new JoinDesc(exprMap, outputColumnNames, + JoinDesc desc = new JoinDesc(keyExprMap, exprMap, outputColumnNames, join.getNoOuterJoin(), joinCondns, filterMap); desc.setReversedExprs(reversedExprs); JoinOperator joinOp = (JoinOperator) OperatorFactory.getAndMakeChild(desc, diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java index 85c9f8f..b434a4a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java @@ -30,4 +30,6 @@ public @interface Explain { String displayName() default ""; boolean normalExplain() default true; + + boolean columnTrack() default false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index 769c70d..feb2582 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -88,7 +88,7 @@ public class FilterDesc implements Serializable { this.sampleDescr = sampleDescr; } - @Explain(displayName = "predicate") + @Explain(displayName = "predicate", columnTrack = true) public org.apache.hadoop.hive.ql.plan.ExprNodeDesc getPredicate() { return predicate; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 9ebbb33..0d510e6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -114,7 +114,7 @@ public class GroupByDesc implements java.io.Serializable { this.mode = mode; } - @Explain(displayName = "keys") + @Explain(displayName = "keys", columnTrack = true) public java.util.ArrayList getKeys() { return keys; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 4a729e8..29742e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -284,7 +284,7 @@ public class HashTableSinkDesc extends JoinDesc implements Serializable { /** * @return the keys */ - @Explain(displayName = "keys") + @Explain(displayName = "keys", columnTrack = true) public Map> getKeys() { return keys; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index df9097d..6f5013c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -46,6 +46,9 @@ public class JoinDesc implements Serializable { private Map skewKeysValuesTables; // alias to key mapping + private Map> keys; + + // alias to value mapping private Map> exprs; // alias to filter mapping @@ -68,9 +71,10 @@ public class JoinDesc implements Serializable { public JoinDesc() { } - public JoinDesc(final Map> exprs, + public JoinDesc(Map> keys, final Map> exprs, List outputColumnNames, final boolean noOuterJoin, final JoinCondDesc[] conds, final Map> filters) { + this.keys = keys; this.exprs = exprs; this.outputColumnNames = outputColumnNames; this.noOuterJoin = noOuterJoin; @@ -83,23 +87,24 @@ public class JoinDesc implements Serializable { } } - public JoinDesc(final Map> exprs, + public JoinDesc(Map> keys, final Map> exprs, List outputColumnNames, final boolean noOuterJoin, final JoinCondDesc[] conds) { - this(exprs, outputColumnNames, noOuterJoin, conds, null); + this(keys, exprs, outputColumnNames, noOuterJoin, conds, null); } - public JoinDesc(final Map> exprs, + public JoinDesc(Map> keys, final Map> exprs, List outputColumnNames) { - this(exprs, outputColumnNames, true, null); + this(keys, exprs, outputColumnNames, true, null); } - public JoinDesc(final Map> exprs, + public JoinDesc(Map> keys, final Map> exprs, List outputColumnNames, final JoinCondDesc[] conds) { - this(exprs, outputColumnNames, true, conds, null); + this(keys, exprs, outputColumnNames, true, conds, null); } public JoinDesc(JoinDesc clone) { + this.keys = clone.keys; this.bigKeysDirMap = clone.bigKeysDirMap; this.conds = clone.conds; this.exprs = clone.exprs; @@ -115,6 +120,7 @@ public class JoinDesc implements Serializable { this.filters = clone.filters; } + @Explain(displayName = "condition expressions", columnTrack = true) public Map> getExprs() { return exprs; } @@ -127,33 +133,13 @@ public class JoinDesc implements Serializable { this.reversedExprs = reversedExprs; } - @Explain(displayName = "condition expressions") - public Map getExprsStringMap() { - if (getExprs() == null) { - return null; - } - - LinkedHashMap ret = new LinkedHashMap(); - - for (Map.Entry> ent : getExprs().entrySet()) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - if (ent.getValue() != null) { - for (ExprNodeDesc expr : ent.getValue()) { - if (!first) { - sb.append(" "); - } - - first = false; - sb.append("{"); - sb.append(expr.getExprString()); - sb.append("}"); - } - } - ret.put(ent.getKey(), sb.toString()); - } + @Explain(displayName = "keys", columnTrack = true) + public Map> getKeys() { + return keys; + } - return ret; + public void setKeys(Map> keys) { + this.keys = keys; } public void setExprs(final Map> exprs) { @@ -167,41 +153,9 @@ public class JoinDesc implements Serializable { * * @return Map from alias to filters on the alias. */ - @Explain(displayName = "filter predicates") - public Map getFiltersStringMap() { - if (getFilters() == null || getFilters().size() == 0) { - return null; - } - - LinkedHashMap ret = new LinkedHashMap(); - boolean filtersPresent = false; - - for (Map.Entry> ent : getFilters().entrySet()) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - if (ent.getValue() != null) { - if (ent.getValue().size() != 0) { - filtersPresent = true; - } - for (ExprNodeDesc expr : ent.getValue()) { - if (!first) { - sb.append(" "); - } - - first = false; - sb.append("{"); - sb.append(expr.getExprString()); - sb.append("}"); - } - } - ret.put(ent.getKey(), sb.toString()); - } - - if (filtersPresent) { - return ret; - } else { - return null; - } + @Explain(displayName = "filter predicates", columnTrack = true) + public Map> getFiltersStringMap() { + return filters; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index ada9826..59a7709 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -36,7 +36,6 @@ import java.util.Map.Entry; public class MapJoinDesc extends JoinDesc implements Serializable { private static final long serialVersionUID = 1L; - private Map> keys; private TableDesc keyTblDesc; private List valueTblDescs; private List valueFilteredTblDescs; @@ -56,7 +55,6 @@ public class MapJoinDesc extends JoinDesc implements Serializable { public MapJoinDesc(MapJoinDesc clone) { super(clone); - this.keys = clone.keys; this.keyTblDesc = clone.keyTblDesc; this.valueTblDescs = clone.valueTblDescs; this.posBigTable = clone.posBigTable; @@ -71,8 +69,7 @@ public class MapJoinDesc extends JoinDesc implements Serializable { final List valueTblDescs,final List valueFilteredTblDescs, List outputColumnNames, final int posBigTable, final JoinCondDesc[] conds, final Map> filters, boolean noOuterJoin) { - super(values, outputColumnNames, noOuterJoin, conds, filters); - this.keys = keys; + super(keys, values, outputColumnNames, noOuterJoin, conds, filters); this.keyTblDesc = keyTblDesc; this.valueTblDescs = valueTblDescs; this.valueFilteredTblDescs = valueFilteredTblDescs; @@ -104,22 +101,6 @@ public class MapJoinDesc extends JoinDesc implements Serializable { } /** - * @return the keys - */ - @Explain(displayName = "keys") - public Map> getKeys() { - return keys; - } - - /** - * @param keys - * the keys to set - */ - public void setKeys(Map> keys) { - this.keys = keys; - } - - /** * @return the position of the big table not in memory */ @Explain(displayName = "Position of Big Table") diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index d7f0476..a62afef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -109,7 +109,7 @@ public class ReduceSinkDesc implements Serializable { this.outputValueColumnNames = outputValueColumnNames; } - @Explain(displayName = "key expressions") + @Explain(displayName = "key expressions", columnTrack = true) public java.util.ArrayList getKeyCols() { return keyCols; } @@ -126,7 +126,7 @@ public class ReduceSinkDesc implements Serializable { this.numDistributionKeys = numKeys; } - @Explain(displayName = "value expressions") + @Explain(displayName = "value expressions", columnTrack = true) public java.util.ArrayList getValueCols() { return valueCols; } @@ -135,7 +135,7 @@ public class ReduceSinkDesc implements Serializable { this.valueCols = valueCols; } - @Explain(displayName = "Map-reduce partition columns") + @Explain(displayName = "Map-reduce partition columns", columnTrack = true) public java.util.ArrayList getPartitionCols() { return partitionCols; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index 97ae10d..09ff874 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -62,7 +62,7 @@ public class SelectDesc implements Serializable { this.selStarNoCompute = selStarNoCompute; } - @Explain(displayName = "expressions") + @Explain(displayName = "expressions", columnTrack = true) public java.util.ArrayList getColList() { return colList; }