Index: itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1571424) +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -1249,6 +1249,7 @@ ".*job_local[0-9_]*.*", ".*USING 'java -cp.*", "^Deleted.*", + ".*DagName:.*" }); public int checkCliDriverResults(String tname) throws Exception { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (revision 1571424) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (working copy) @@ -154,17 +154,24 @@ public JSONObject getJSONPlan(PrintStream out, ExplainWork work) throws Exception { + return getJSONPlan(out, work.getAstStringTree(), work.getRootTasks(), work.getFetchTask(), + work.isFormatted(), work.getExtended(), work.isAppendTaskType()); + } + + public JSONObject getJSONPlan(PrintStream out, String ast, List> tasks, Task fetchTask, + boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception { + // If the user asked for a formatted output, dump the json output // in the output stream JSONObject outJSONObject = new JSONObject(); - boolean jsonOutput = work.isFormatted(); + if (jsonOutput) { out = null; } // Print out the parse AST - if (work.getAstStringTree() != null && work.getExtended()) { - String jsonAST = outputAST(work.getAstStringTree(), out, jsonOutput, 0); + if (ast != null && isExtended) { + String jsonAST = outputAST(ast, out, jsonOutput, 0); if (out != null) { out.println(); } @@ -173,16 +180,15 @@ outJSONObject.put("ABSTRACT SYNTAX TREE", jsonAST); } } - List> tasks = work.getRootTasks(); List ordered = StageIDsRearranger.getExplainOrder(conf, tasks); - Task fetchTask = work.getFetchTask(); + if (fetchTask != null) { fetchTask.setRootTask(true); // todo HIVE-3925 ordered.add(fetchTask); } - JSONObject jsonDependencies = outputDependencies(out, work, ordered); + JSONObject jsonDependencies = outputDependencies(out, jsonOutput, appendTaskType, ordered); if (out != null) { out.println(); @@ -193,7 +199,8 @@ } // Go over all the tasks and dump out the plans - JSONObject jsonPlan = outputStagePlans(out, work, ordered); + JSONObject jsonPlan = outputStagePlans(out, ordered, + jsonOutput, isExtended); if (jsonOutput) { outJSONObject.put("STAGE PLANS", jsonPlan); @@ -748,10 +755,10 @@ return jsonOutput ? treeString : null; } - public JSONObject outputDependencies(PrintStream out, ExplainWork work, List tasks) + public JSONObject outputDependencies(PrintStream out, boolean jsonOutput, + boolean appendTaskType, List tasks) throws Exception { - boolean jsonOutput = work.isFormatted(); - boolean appendTaskType = work.isAppendTaskType(); + if (out != null) { out.println("STAGE DEPENDENCIES:"); } @@ -767,9 +774,10 @@ return jsonOutput ? json : null; } - public JSONObject outputStagePlans(PrintStream out, ExplainWork work, List tasks) + public JSONObject outputStagePlans(PrintStream out, List tasks, + boolean jsonOutput, boolean isExtended) throws Exception { - boolean jsonOutput = work.isFormatted(); + if (out != null) { out.println("STAGE PLANS:"); } @@ -776,7 +784,7 @@ JSONObject json = jsonOutput ? new JSONObject() : null; for (Task task : tasks) { - outputPlan(task, out, json, work.getExtended(), jsonOutput, 2); + outputPlan(task, out, json, isExtended, jsonOutput, 2); } return jsonOutput ? json : null; } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (revision 1571424) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (working copy) @@ -207,9 +207,7 @@ FileSystem fs = tezDir.getFileSystem(conf); // the name of the dag is what is displayed in the AM/Job UI - DAG dag = new DAG( - Utilities.abbreviate(HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING), - HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEJOBNAMELENGTH))); + DAG dag = new DAG(work.getName()); for (BaseWork w: ws) { Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 1571424) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy) @@ -1250,7 +1250,7 @@ } else { cplan = createMRWorkForMergingFiles(conf, tsMerge, fsInputDesc); if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { - work = new TezWork(); + work = new TezWork(conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); cplan.setName("Merge"); ((TezWork)work).add(cplan); } else { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java (revision 1571424) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java (working copy) @@ -131,7 +131,8 @@ this.rootTasks = rootTasks; this.inputs = inputs; this.outputs = outputs; - this.currentTask = (TezTask) TaskFactory.get(new TezWork(), conf); + this.currentTask = (TezTask) TaskFactory.get( + new TezWork(conf.getVar(HiveConf.ConfVars.HIVEQUERYID)), conf); this.leafOperatorToFollowingWork = new HashMap, BaseWork>(); this.linkOpWithWorkMap = new HashMap, List>(); this.linkWorkWithReduceSinkMap = new HashMap>(); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (revision 1571424) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (working copy) @@ -51,6 +51,8 @@ private static transient final Log LOG = LogFactory.getLog(TezWork.class); + private static int counter; + private final String name; private final Set roots = new HashSet(); private final Set leaves = new HashSet(); private final Map> workGraph = new HashMap>(); @@ -58,6 +60,15 @@ private final Map, EdgeType> edgeProperties = new HashMap, EdgeType>(); + public TezWork(String name) { + this.name = name + ":" + (++counter); + } + + @Explain(displayName = "DagName") + public String getName() { + return name; + } + /** * getWorkMap returns a map of "vertex name" to BaseWork */ Index: ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (revision 1571424) +++ ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (working copy) @@ -112,7 +112,7 @@ } }); - work = new TezWork(); + work = new TezWork(""); mws = new MapWork[] { new MapWork(), new MapWork()}; rws = new ReduceWork[] { new ReduceWork(), new ReduceWork() }; @@ -194,7 +194,7 @@ @Test public void testEmptyWork() throws IllegalArgumentException, IOException, Exception { - DAG dag = task.build(conf, new TezWork(), path, appLr, new Context(conf)); + DAG dag = task.build(conf, new TezWork(""), path, appLr, new Context(conf)); assertEquals(dag.getVertices().size(), 0); } Index: ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java (revision 1571424) +++ ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java (working copy) @@ -34,7 +34,7 @@ @Before public void setup() throws Exception { nodes = new LinkedList(); - work = new TezWork(); + work = new TezWork(""); addWork(5); } Index: ql/src/test/results/clientpositive/tez/auto_join0.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join0.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/auto_join0.q.out (working copy) @@ -31,6 +31,7 @@ Map 1 <- Map 4 (BROADCAST_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/auto_join1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/auto_join1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/auto_join1.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Map 2 <- Map 1 (BROADCAST_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/bucket2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/bucket2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/bucket2.q.out (working copy) @@ -39,6 +39,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -196,6 +197,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/bucket3.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/bucket3.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/bucket3.q.out (working copy) @@ -43,6 +43,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -221,6 +222,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/bucket4.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/bucket4.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/bucket4.q.out (working copy) @@ -39,6 +39,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -197,6 +198,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/count.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/count.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/count.q.out (working copy) @@ -37,6 +37,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -108,6 +109,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -174,6 +176,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -239,6 +242,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/cross_join.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/cross_join.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/cross_join.q.out (working copy) @@ -13,6 +13,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -72,6 +73,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -131,6 +133,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/ctas.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/ctas.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/ctas.q.out (working copy) @@ -32,6 +32,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -171,6 +172,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -310,6 +312,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -513,6 +516,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -694,6 +698,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out (working copy) @@ -39,6 +39,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -196,6 +197,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out (working copy) @@ -139,6 +139,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/groupby1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/groupby1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/groupby1.q.out (working copy) @@ -21,6 +21,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/groupby2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/groupby2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/groupby2.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/groupby3.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/groupby3.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/groupby3.q.out (working copy) @@ -41,6 +41,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/having.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/having.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/having.q.out (working copy) @@ -11,6 +11,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -91,6 +92,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -469,6 +471,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -738,6 +741,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -936,6 +940,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/insert1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/insert1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/insert1.q.out (working copy) @@ -33,6 +33,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -86,6 +87,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -154,6 +156,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -207,6 +210,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -268,6 +272,7 @@ STAGE PLANS: Stage: Stage-2 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/insert_into1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/insert_into1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/insert_into1.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -117,6 +118,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -231,6 +233,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/insert_into2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/insert_into2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/insert_into2.q.out (working copy) @@ -26,6 +26,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -158,6 +159,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -275,6 +277,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/join0.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/join0.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/join0.q.out (working copy) @@ -24,6 +24,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -108,7 +109,7 @@ (SELECT * FROM src WHERE src.key < 10) src2 SORT BY k1, v1, k2, v2 POSTHOOK: type: QUERY -{"STAGE PLANS":{"Stage-1":{"Tez":{"Vertices:":{"Reducer 2":{"Reduce Operator Tree:":{"Join Operator":{"outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"Select Operator":{"expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"children":{"Reduce Output Operator":{"sort order:":"++++","value expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","Statistics:":"Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE","key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)"}},"Statistics:":"Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE","condition map:":[{"":"Inner Join 0 to 1"}],"condition expressions:":{"1":"{VALUE._col0} {VALUE._col1}","0":"{VALUE._col0} {VALUE._col1}"}}}},"Reducer 3":{"Reduce Operator Tree:":{"Extract":{"children":{"File Output Operator":{"Statistics:":"Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE","compressed:":"false","table:":{"serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe","input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}},"Statistics:":"Num rows: 9 Data size: 1983 Basic stats: COMPLETE Column stats: NONE"}}},"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE"}}]},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"children":{"Reduce Output Operator":{"sort order:":"","value expressions:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE"}},"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE"}}]}},"Edges:":{"Reducer 2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 4","type":"SIMPLE_EDGE"}],"Reducer 3":{"parent":"Reducer 2","type":"SIMPLE_EDGE"}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}}} +#### A masked pattern was here #### PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM (SELECT * FROM src WHERE src.key < 10) src1 Index: ql/src/test/results/clientpositive/tez/join1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/join1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/join1.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/limit_pushdown.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/limit_pushdown.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/limit_pushdown.q.out (working copy) @@ -17,6 +17,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -95,6 +96,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -173,6 +175,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -268,6 +271,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -363,6 +367,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -453,6 +458,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -548,6 +554,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -643,6 +650,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -703,6 +711,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -818,6 +827,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -963,6 +973,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -1052,6 +1063,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -1212,6 +1224,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out (working copy) @@ -53,6 +53,7 @@ STAGE PLANS: Stage: Stage-2 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out (working copy) @@ -40,6 +40,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/load_dyn_part3.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/load_dyn_part3.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/load_dyn_part3.q.out (working copy) @@ -42,6 +42,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out (working copy) @@ -15,6 +15,7 @@ Tez Edges: Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -93,6 +94,7 @@ Edges: Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/mapreduce1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mapreduce1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/mapreduce1.q.out (working copy) @@ -30,6 +30,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/mapreduce2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mapreduce2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/mapreduce2.q.out (working copy) @@ -28,6 +28,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/merge1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/merge1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/merge1.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -483,6 +484,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -559,6 +561,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/merge2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/merge2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/merge2.q.out (working copy) @@ -22,6 +22,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -483,6 +484,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -559,6 +561,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/metadata_only_queries.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/metadata_only_queries.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/metadata_only_queries.q.out (working copy) @@ -290,6 +290,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -391,6 +392,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -922,6 +924,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/mrr.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/mrr.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/mrr.q.out (working copy) @@ -14,6 +14,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -403,6 +404,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -818,6 +820,7 @@ Map 2 <- Map 1 (BROADCAST_EDGE) Reducer 3 <- Map 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -1262,6 +1265,7 @@ Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -1641,6 +1645,7 @@ Edges: Map 1 <- Reducer 3 (BROADCAST_EDGE) Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/sample1.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/sample1.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/sample1.q.out (working copy) @@ -63,6 +63,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/subquery_exists.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/subquery_exists.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/subquery_exists.q.out (working copy) @@ -27,6 +27,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/subquery_in.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/subquery_in.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/subquery_in.q.out (working copy) @@ -105,6 +105,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -220,6 +221,7 @@ Tez Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -345,6 +347,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -491,6 +494,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -644,6 +648,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -806,6 +811,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/tez_dml.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/tez_dml.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/tez_dml.q.out (working copy) @@ -17,6 +17,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -433,6 +434,7 @@ STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: @@ -867,6 +869,7 @@ STAGE PLANS: Stage: Stage-2 Tez +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/tez_join_tests.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/tez_join_tests.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/tez_join_tests.q.out (working copy) @@ -16,6 +16,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out (working copy) @@ -16,6 +16,7 @@ Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union2.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union2.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union2.q.out (working copy) @@ -21,6 +21,7 @@ Map 1 <- Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union3.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union3.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union3.q.out (working copy) @@ -48,6 +48,7 @@ Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union4.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union4.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union4.q.out (working copy) @@ -33,6 +33,7 @@ Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 5 <- Map 4 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union5.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union5.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union5.q.out (working copy) @@ -23,6 +23,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union6.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union6.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union6.q.out (working copy) @@ -33,6 +33,7 @@ Edges: Map 4 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union7.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union7.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union7.q.out (working copy) @@ -23,6 +23,7 @@ Map 5 <- Union 3 (CONTAINS) Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union8.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union8.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union8.q.out (working copy) @@ -23,6 +23,7 @@ Map 1 <- Union 2 (CONTAINS) Map 3 <- Union 2 (CONTAINS) Map 4 <- Union 2 (CONTAINS) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: Index: ql/src/test/results/clientpositive/tez/union9.q.out =================================================================== --- ql/src/test/results/clientpositive/tez/union9.q.out (revision 1571424) +++ ql/src/test/results/clientpositive/tez/union9.q.out (working copy) @@ -24,6 +24,7 @@ Map 4 <- Union 2 (CONTAINS) Map 5 <- Union 2 (CONTAINS) Reducer 3 <- Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: