Index: ql/src/test/results/clientpositive/explain_dependency.q.out =================================================================== --- ql/src/test/results/clientpositive/explain_dependency.q.out (revision 1421185) +++ ql/src/test/results/clientpositive/explain_dependency.q.out (working copy) @@ -54,14 +54,14 @@ EXPLAIN DEPENDENCY SELECT key, count(1) FROM srcpart WHERE ds IS NOT NULL GROUP BY key POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@srcpart","tableDepth":"0","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT key, count(1) FROM (SELECT key, value FROM src) subq1 GROUP BY key PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN DEPENDENCY SELECT key, count(1) FROM (SELECT key, value FROM src) subq1 GROUP BY key POSTHOOK: type: QUERY -{"input_partitions":[],"input_tables":[{"tablename":"default@src","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[],"input_tables":[{"tablename":"default@src","tableDepth":"0","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM ( SELECT key, value FROM src UNION ALL SELECT key, value FROM srcpart WHERE ds IS NOT NULL @@ -72,33 +72,33 @@ SELECT key, value FROM src UNION ALL SELECT key, value FROM srcpart WHERE ds IS NOT NULL ) S1 POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"},{"tablename":"default@src","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@srcpart","tableDepth":"0","tabletype":"MANAGED_TABLE"},{"tablename":"default@src","tableDepth":"0","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT S1.key, S2.value FROM src S1 JOIN srcpart S2 ON S1.key = S2.key WHERE ds IS NOT NULL PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN DEPENDENCY SELECT S1.key, S2.value FROM src S1 JOIN srcpart S2 ON S1.key = S2.key WHERE ds IS NOT NULL POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"},{"tablename":"default@src","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"0","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@srcpart","tableDepth":"0","tabletype":"MANAGED_TABLE"},{"tablename":"default@src","tableDepth":"0","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: -- With views EXPLAIN DEPENDENCY SELECT * FROM V1 PREHOOK: type: QUERY POSTHOOK: query: -- With views EXPLAIN DEPENDENCY SELECT * FROM V1 POSTHOOK: type: QUERY -{"input_partitions":[],"input_tables":[{"tablename":"default@v1","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[],"input_tables":[{"tablename":"default@v1","tableDepth":"0","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tableDepth":"1","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V2 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V2 POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@v2","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"1","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"1","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"1","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"1","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@v2","tableDepth":"0","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@srcpart","tableDepth":"1","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V3 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V3 POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@v3","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v2","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tabletype":"MANAGED_TABLE"},{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@v3","tableDepth":"0","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v2","tableDepth":"1","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tableDepth":"1","tabletype":"MANAGED_TABLE"},{"tablename":"default@srcpart","tableDepth":"2","tabletype":"MANAGED_TABLE"}]} PREHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V4 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V4 POSTHOOK: type: QUERY -{"input_partitions":["default@srcpart@ds=2008-04-08/hr=11","default@srcpart@ds=2008-04-08/hr=12","default@srcpart@ds=2008-04-09/hr=11","default@srcpart@ds=2008-04-09/hr=12"],"input_tables":[{"tablename":"default@v4","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v2","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v1","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tabletype":"MANAGED_TABLE"},{"tablename":"default@srcpart","tabletype":"MANAGED_TABLE"}]} +{"input_partitions":[{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-08/hr=11"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-08/hr=12"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-09/hr=11"},{"partitionDepth":"2","partitionName":"default@srcpart@ds=2008-04-09/hr=12"}],"input_tables":[{"tablename":"default@v4","tableDepth":"0","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v2","tableDepth":"1","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@v1","tableDepth":"1","tabletype":"VIRTUAL_VIEW"},{"tablename":"default@src","tableDepth":"2","tabletype":"MANAGED_TABLE"},{"tablename":"default@src","tableDepth":"1","tabletype":"MANAGED_TABLE"},{"tablename":"default@srcpart","tableDepth":"2","tabletype":"MANAGED_TABLE"}]} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 1421185) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy) @@ -601,6 +601,24 @@ setTaskPlan(alias_id, topOp, plan, local, opProcCtx, null); } + private static int getViewDepth(String alias_id, Map viewAliasToInput) { + String[] aliases = alias_id.split(":"); + + String currentAlias = null; + ReadEntity currentInput = null; + for (int pos = 0; pos < aliases.length; pos++) { + currentAlias = currentAlias == null ? aliases[pos] : currentAlias + ":" + aliases[pos]; + ReadEntity input = viewAliasToInput.get(currentAlias); + if (input == null) { + break; + } + currentInput = input; + } + + return (currentInput == null) ? 0 : currentInput.getDepth() + 1; + } + + /** * set the current task in the mapredWork. * @@ -703,11 +721,19 @@ boolean isFirstPart = true; boolean emptyInput = true; boolean singlePartition = (parts.size() == 1); + + // Track the dependencies for the view. Consider a query like: select * from V; + // where V is a view of the form: select * from T + // The dependencies should include V at depth 0, and T at depth 1 (inferred). + int viewDepth = getViewDepth(alias_id, parseCtx.getViewAliasToInput()); + // The table should also be considered a part of inputs, even if the table is a + // partitioned table and whether any partition is selected or not + inputs.add(new ReadEntity(parseCtx.getTopToTable().get(topOp), viewDepth)); for (Partition part : parts) { if (part.getTable().isPartitioned()) { - inputs.add(new ReadEntity(part)); + inputs.add(new ReadEntity(part, viewDepth)); } else { - inputs.add(new ReadEntity(part.getTable())); + inputs.add(new ReadEntity(part.getTable(), viewDepth)); } // Later the properties have to come from the partition as opposed Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (revision 1421185) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (working copy) @@ -78,34 +78,30 @@ JSONObject outJSONObject = new JSONObject(); List> inputTableInfo = new ArrayList>(); - Set inputPartitions = new HashSet(); - Set inputTables = new HashSet(); - Table table = null; + List> inputPartitionInfo = new ArrayList>(); for (ReadEntity input: work.getInputs()) { switch (input.getType()) { case TABLE: - table = input.getTable(); + Table table = input.getTable(); + Map tableInfo = new HashMap(); + tableInfo.put("tablename", table.getCompleteName()); + tableInfo.put("tabletype", table.getTableType().toString()); + tableInfo.put("tableDepth", String.valueOf(input.getDepth())); + inputTableInfo.add(tableInfo); break; case PARTITION: - inputPartitions.add(input.getPartition().getCompleteName()); - table = input.getPartition().getTable(); + Map partitionInfo = new HashMap(); + partitionInfo.put("partitionName", input.getPartition().getCompleteName()); + partitionInfo.put("partitionDepth", String.valueOf(input.getDepth())); + inputPartitionInfo.add(partitionInfo); break; default: - table = null; break; } - - if (table != null && !inputTables.contains(table.getCompleteName())) { - Map tableInfo = new HashMap(); - tableInfo.put("tablename", table.getCompleteName()); - tableInfo.put("tabletype", table.getTableType().toString()); - inputTableInfo.add(tableInfo); - inputTables.add(table.getCompleteName()); - } } outJSONObject.put("input_tables", inputTableInfo); - outJSONObject.put("input_partitions", inputPartitions); + outJSONObject.put("input_partitions", inputPartitionInfo); return outJSONObject; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1421185) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -220,6 +220,8 @@ private final String autogenColAliasPrfxLbl; private final boolean autogenColAliasPrfxIncludeFuncName; + private final Map viewAliasToInput = new HashMap(); + //Max characters when auto generating the column name with func name private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20; @@ -302,7 +304,7 @@ loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, - opToPartToSkewedPruner); + opToPartToSkewedPruner, viewAliasToInput); } @SuppressWarnings("nls") @@ -941,16 +943,24 @@ } private void getMetaData(QBExpr qbexpr) throws SemanticException { + getMetaData(qbexpr, 0); + } + + private void getMetaData(QBExpr qbexpr, int viewDepth) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { - getMetaData(qbexpr.getQB()); + getMetaData(qbexpr.getQB(), viewDepth); } else { - getMetaData(qbexpr.getQBExpr1()); - getMetaData(qbexpr.getQBExpr2()); + getMetaData(qbexpr.getQBExpr1(), viewDepth); + getMetaData(qbexpr.getQBExpr2(), viewDepth); } } + public void getMetaData(QB qb) throws SemanticException { + getMetaData(qb, 0); + } + @SuppressWarnings("nls") - public void getMetaData(QB qb) throws SemanticException { + public void getMetaData(QB qb, int viewDepth) throws SemanticException { try { LOG.info("Get metadata for source tables"); @@ -1004,7 +1014,9 @@ aliasToViewName.put(alias, fullViewName); // This is the last time we'll see the Table objects for views, so add it to the inputs // now - inputs.add(new ReadEntity(tab)); + ReadEntity viewInput = new ReadEntity(tab, viewDepth); + inputs.add(viewInput); + viewAliasToInput.put(getAliasId(alias, qb), viewInput); continue; } @@ -1038,7 +1050,7 @@ viewsExpanded.add(aliasToViewName.get(alias)); } QBExpr qbexpr = qb.getSubqForAlias(alias); - getMetaData(qbexpr); + getMetaData(qbexpr, wasView ? viewDepth + 1 : viewDepth); if (wasView) { viewsExpanded.remove(viewsExpanded.size()-1); } @@ -8214,7 +8226,7 @@ loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, - opToPartToSkewedPruner); + opToPartToSkewedPruner, viewAliasToInput); // Generate table access stats if required if (HiveConf.getBoolVar(this.conf, HiveConf.ConfVars.HIVE_STATS_COLLECT_TABLEKEYS) == true) { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (revision 1421185) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (working copy) @@ -85,6 +85,7 @@ // reducer private Map> groupOpToInputTables; private Map prunedPartitions; + private Map viewAliasToInput; /** * The lineage information. @@ -169,7 +170,8 @@ GlobalLimitCtx globalLimitCtx, HashMap nameToSplitSample, HashSet semanticInputs, List> rootTasks, - Map> opToPartToSkewedPruner) { + Map> opToPartToSkewedPruner, + Map viewAliasToInput) { this.conf = conf; this.qb = qb; this.ast = ast; @@ -196,6 +198,7 @@ this.semanticInputs = semanticInputs; this.rootTasks = rootTasks; this.opToPartToSkewedPruner = opToPartToSkewedPruner; + this.viewAliasToInput = viewAliasToInput; } /** @@ -578,4 +581,7 @@ this.opToPartToSkewedPruner = opToPartToSkewedPruner; } + public Map getViewAliasToInput() { + return viewAliasToInput; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java (revision 1421185) +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/ReadEntity.java (working copy) @@ -29,6 +29,9 @@ */ public class ReadEntity extends Entity implements Serializable { + // For views, the entities can be nested - by default, entities are at the top level + private int depth = 0; + /** * For serialization only. */ @@ -46,6 +49,11 @@ super(t); } + public ReadEntity(Table t, int depth) { + super(t); + this.depth = depth; + } + /** * Constructor given a partiton. * @@ -56,6 +64,15 @@ super(p); } + public ReadEntity(Partition p, int depth) { + super(p); + this.depth = depth; + } + + public int getDepth() { + return depth; + } + /** * Equals function. */ @@ -67,7 +84,7 @@ if (o instanceof ReadEntity) { ReadEntity ore = (ReadEntity) o; - return (toString().equalsIgnoreCase(ore.toString())); + return (toString().equalsIgnoreCase(ore.toString()) && (ore.depth == depth)); } else { return false; }