diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java index b87d9045d0..39c18b519e 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java @@ -288,6 +288,13 @@ public LineProcessingResult processLine(String line, String tname) { ppm.add(new PatternReplacementPair(Pattern.compile("for Spark session.*?:"), "#SPARK_SESSION_ID#:")); + ppm.add(new PatternReplacementPair(Pattern.compile("\"CBOPlan\".*,\"optimizedSQL\""), + "\"CBOPlan\":\"###Masked###\",\"optimizedSQL\"")); + ppm.add(new PatternReplacementPair(Pattern.compile("\"DagId:\".*,\"Edges:\""), + "\"DagId:\":\"###Masked###\",\"Edges:\"")); + ppm.add(new PatternReplacementPair(Pattern.compile("\"DagName:\".*,\"Vertices:\""), + "\"DagName:\":\"###Masked###\",\"Vertices:\"")); + ppm.add(new PatternReplacementPair(Pattern.compile("rowcount = [0-9]+(\\.[0-9]+(E[0-9]+)?)?, cumulative cost = \\{.*\\}, id = [0-9]*"), "rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###")); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 4f14fa59c1..3a20130820 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1047,7 +1047,7 @@ private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan, if (conf.getBoolVar(ConfVars.HIVE_SERVER2_WEBUI_SHOW_GRAPH)) { JSONObject jsonPlan = task.getJSONPlan( null, rootTasks, sem.getFetchTask(), true, true, true, sem.getCboInfo(), - plan.getOptimizedQueryString()); + plan.getOptimizedCBOPlan(), plan.getOptimizedQueryString()); if (jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES) != null && jsonPlan.getJSONObject(ExplainTask.STAGE_DEPENDENCIES).length() <= conf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_GRAPH_SIZE)) { @@ -1057,7 +1057,7 @@ private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan, } } else { task.getJSONPlan(ps, rootTasks, sem.getFetchTask(), false, true, true, sem.getCboInfo(), - plan.getOptimizedQueryString()); + plan.getOptimizedCBOPlan(), plan.getOptimizedQueryString()); ret = baos.toString(); } } catch (Exception e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index b9d1e0b93f..8223d6b823 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -237,12 +237,12 @@ public JSONObject getJSONPlan(PrintStream out, ExplainWork work) throws Exception { return getJSONPlan(out, work.getRootTasks(), work.getFetchTask(), work.isFormatted(), work.getExtended(), work.isAppendTaskType(), work.getCboInfo(), - work.getOptimizedSQL()); + work.getCboPlan(), work.getOptimizedSQL()); } public JSONObject getJSONPlan(PrintStream out, List> tasks, Task fetchTask, boolean jsonOutput, boolean isExtended, boolean appendTaskType, String cboInfo, - String optimizedSQL) throws Exception { + String cboPlan, String optimizedSQL) throws Exception { // If the user asked for a formatted output, dump the json output // in the output stream @@ -252,6 +252,15 @@ public JSONObject getJSONPlan(PrintStream out, List> tasks, Task fetc out = null; } + if (cboPlan != null) { + if (jsonOutput) { + outJSONObject.put("CBOPlan", cboPlan); + } else { + out.print("CBO PLAN:"); + out.println(cboPlan); + } + } + if (optimizedSQL != null) { if (jsonOutput) { outJSONObject.put("optimizedSQL", optimizedSQL); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index bc150fbe87..a7cdddbf08 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -554,7 +554,10 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // Do not include join cost this.ctx.setCalcitePlan(RelOptUtil.toString(newPlan)); } - } else if (explainConfig.isExtended() || explainConfig.isFormatted()) { + } else if (explainConfig.isFormatted()) { + this.ctx.setCalcitePlan(HiveRelOptUtil.toJsonString(newPlan)); + this.ctx.setOptimizedSql(getOptimizedSql(newPlan)); + } else if (explainConfig.isExtended()) { this.ctx.setOptimizedSql(getOptimizedSql(newPlan)); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java index a0c0413431..8ba612ebce 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainTask.java @@ -218,7 +218,7 @@ public void testGetJSONPlan() throws Exception { JsonNode result = objectMapper.readTree(uut.getJSONPlan(null, tasks, null, true, - false, false, "Plan Optimized by CBO", null).toString()); + false, false, "Plan Optimized by CBO", null, null).toString()); JsonNode expected = objectMapper.readTree("{\"cboInfo\":\"Plan Optimized by CBO\", \"STAGE DEPENDENCIES\":{\"mockTaskId\":" + "{\"ROOT STAGE\":\"TRUE\",\"BACKUP STAGE\":\"backup-id-mock\"}},\"STAGE PLANS\":" + "{\"mockTaskId\":{}}}"); diff --git a/ql/src/test/results/clientpositive/concat_op.q.out b/ql/src/test/results/clientpositive/concat_op.q.out index 9c4fca2442..3e698a5c6a 100644 --- a/ql/src/test/results/clientpositive/concat_op.q.out +++ b/ql/src/test/results/clientpositive/concat_op.q.out @@ -311,7 +311,7 @@ POSTHOOK: query: explain formatted select key || value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"optimizedSQL":"SELECT `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(key, value) (type: string)","columnExprMap:":{"_col0":"concat(key, value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(key, value) (type: string)","columnExprMap:":{"_col0":"concat(key, value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} PREHOOK: query: explain formatted select key || value || key from src PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -320,7 +320,7 @@ POSTHOOK: query: explain formatted select key || value || key from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"optimizedSQL":"SELECT `key` || `value` || `key` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(key, value), key) (type: string)","columnExprMap:":{"_col0":"concat(concat(key, value), key)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `key` || `value` || `key` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(key, value), key) (type: string)","columnExprMap:":{"_col0":"concat(concat(key, value), key)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} PREHOOK: query: explain formatted select key || value || key || value from src PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -329,4 +329,4 @@ POSTHOOK: query: explain formatted select key || value || key || value from src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"optimizedSQL":"SELECT `key` || `value` || `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(concat(key, value), key), value) (type: string)","columnExprMap:":{"_col0":"concat(concat(concat(key, value), key), value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `key` || `value` || `key` || `value` AS `_o__c0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"concat(concat(concat(key, value), key), value) (type: string)","columnExprMap:":{"_col0":"concat(concat(concat(key, value), key), value)"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_3","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_4"}}}}}}],"Execution mode:":"vectorized"}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_5"}}}}}} diff --git a/ql/src/test/results/clientpositive/input4.q.out b/ql/src/test/results/clientpositive/input4.q.out index c8951e8e9e..99c8cca0f6 100644 --- a/ql/src/test/results/clientpositive/input4.q.out +++ b/ql/src/test/results/clientpositive/input4.q.out @@ -53,7 +53,7 @@ SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4_n0 AS Input4Alias POSTHOOK: type: QUERY POSTHOOK: Input: default@input4_n0 #### A masked pattern was here #### -{"optimizedSQL":"SELECT `value`, `key`\nFROM `default`.`input4_n0`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","columns:":["value","key"],"database:":"default","Statistics:":"Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE","table:":"input4_n0","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","columnExprMap:":{"_col0":"value","_col1":"key"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `value`, `key`\nFROM `default`.`input4_n0`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","columns:":["value","key"],"database:":"default","Statistics:":"Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE","table:":"input4_n0","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","columnExprMap:":{"_col0":"value","_col1":"key"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4_n0 AS Input4Alias PREHOOK: type: QUERY PREHOOK: Input: default@input4_n0 diff --git a/ql/src/test/results/clientpositive/llap/join0.q.out b/ql/src/test/results/clientpositive/llap/join0.q.out index 227278c1d4..cdd9a912d8 100644 --- a/ql/src/test/results/clientpositive/llap/join0.q.out +++ b/ql/src/test/results/clientpositive/llap/join0.q.out @@ -127,6 +127,7 @@ SELECT src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +{"cboInfo":"Plan not optimized by CBO because the statement has sort by","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Reducer 2":[{"parent":"Map 1","type":"XPROD_EDGE"},{"parent":"Map 4","type":"XPROD_EDGE"}],"Reducer 3":{"parent":"Reducer 2","type":"SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_15"]}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_15"]}}}}}}}}],"Execution mode:":"llap","LLAP IO:":"no inputs"},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MERGEJOIN_15","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_10","outputname:":"Reducer 3","outputOperator:":["SEL_11"]}}}}},"Reducer 3":{"Execution mode:":"llap","Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 9809936 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_16"}}}}}} Warning: Shuffle Join MERGEJOIN[15][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/llap/tez_union.q.out b/ql/src/test/results/clientpositive/llap/tez_union.q.out index bae4176664..f729b80a25 100644 --- a/ql/src/test/results/clientpositive/llap/tez_union.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -1524,6 +1524,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@table1_n3 POSTHOOK: Input: default@view_n123456789 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`table1_n3`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"table1_n3","database:":"default","properties:":{"insideView":"TRUE"},"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE","table:":"table1_n3","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"columnExprMap:":{"BLOCK__OFFSET__INSIDE__FILE":"BLOCK__OFFSET__INSIDE__FILE","INPUT__FILE__NAME":"INPUT__FILE__NAME","ROW__ID":"ROW__ID","emp_id":"emp_id","emp_name":"emp_name"},"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"GBY_8","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_9","outputname:":"Reducer 2","outputOperator:":["GBY_10"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"no inputs"},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"GBY_10","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_11"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_12"}}}}}} PREHOOK: query: drop table table2_n1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table2_n1 @@ -1564,3 +1565,4 @@ POSTHOOK: Input: default@table1_n3 POSTHOOK: Input: default@table2_n1 POSTHOOK: Input: default@view_n123456789 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `emp_id`\nFROM `default`.`table1_n3`\nUNION ALL\nSELECT `emp_id`\nFROM `default`.`table2_n1`)","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Union 2","type":"CONTAINS"},"Map 4":{"parent":"Union 2","type":"CONTAINS"},"Reducer 3":{"parent":"Union 2","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"table1_n3","database:":"default","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE","table:":"table1_n3","OperatorId:":"TS_12","children":{"Select Operator":{"columnExprMap:":{"_col0":"emp_id"},"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Select Operator":{"Statistics:":"Num rows: 2 Data size: 4 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"SEL_25","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"GBY_26","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_27","outputname:":"Reducer 3","outputOperator:":["GBY_28"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"no inputs"},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"table2_n1","database:":"default","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"table2_n1","OperatorId:":"TS_18","children":{"Select Operator":{"columnExprMap:":{"_col0":"emp_id"},"Statistics:":"Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_30","children":{"Select Operator":{"Statistics:":"Num rows: 2 Data size: 4 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"SEL_31","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"GBY_32","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_33","outputname:":"Reducer 3","outputOperator:":["GBY_28"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"unknown"},"Reducer 3":{"Execution mode:":"vectorized, llap","Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_29"}}}}},"Union 2":{"Vertex:":"Union 2"}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_34"}}}}}} diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index 973be7ce53..137ff3e264 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -248,6 +248,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 3","type":"BROADCAST_EDGE"},{"parent":"Map 4","type":"BROADCAST_EDGE"}],"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_44","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[6]","bigTableValueColumns:":["6:string"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col1"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 25 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_45","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 31 Data size: 248 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_46","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_47","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_48","outputname:":"Reducer 2","outputOperator:":["GBY_49"]}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 2:int)"},"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_38","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2]"},"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_39","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["2:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_40","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 6:string)"},"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_41","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6]"},"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_42","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkStringOperator","keyColumns:":["6:string"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_43","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_49","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_50"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_51"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -291,6 +292,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 3","type":"BROADCAST_EDGE"},{"parent":"Map 4","type":"BROADCAST_EDGE"}],"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_44","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["7:string"],"bigTableRetainColumnNums:":"[6]","bigTableValueColumns:":["6:string"],"className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_45","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 25 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_46","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_47","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_48","outputname:":"Reducer 2","outputOperator:":["GBY_49"]}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 7:string)"},"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_38","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[7]"},"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_39","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkStringOperator","keyColumns:":["7:string"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_40","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 6:string)"},"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_41","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6]"},"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_42","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkStringOperator","keyColumns:":["6:string"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_43","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_49","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_50"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_51"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -334,6 +336,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 3","type":"BROADCAST_EDGE"},{"parent":"Map 4","type":"BROADCAST_EDGE"}],"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_54","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["3:bigint","7:string"],"bigTableRetainColumnNums:":"[2, 6]","bigTableValueColumns:":["2:int","6:string"],"className:":"VectorMapJoinOuterMultiKeyOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["2:int","6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_55","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int","6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterMultiKeyOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 24 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_56","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_57","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_58","outputname:":"Reducer 2","outputOperator:":["GBY_59"]}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cstring2 is not null and cbigint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), SelectColumnIsNotNull(col 3:bigint))"},"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_48","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[3, 7]"},"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_49","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1"},"key expressions:":"_col0 (type: bigint), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: bigint), _col1 (type: string)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkMultiKeyOperator","keyColumns:":["3:bigint","7:string"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_50","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[3, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cstring1 is not null and cint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"FilterExprAndExpr(children: SelectColumnIsNotNull(col 6:string), SelectColumnIsNotNull(col 2:int))"},"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_51","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_52","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1"},"key expressions:":"_col0 (type: int), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: int), _col1 (type: string)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkMultiKeyOperator","keyColumns:":["2:int","6:string"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_53","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_59","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_60"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_61"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out index 561b66410c..f7b304b6b4 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out @@ -262,6 +262,7 @@ left outer join small_alltypesorc_b cd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 2","type":"BROADCAST_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_22","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","bigTableValueColumns:":["0:tinyint","1:smallint","2:int","3:bigint","4:float","5:double","6:string","7:string","8:timestamp","9:timestamp","10:boolean","11:boolean"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"outerSmallTableKeyMapping:":["2 -> 15"],"projectedOutput:":["0:tinyint","1:smallint","2:int","3:bigint","4:float","5:double","6:string","7:string","8:timestamp","9:timestamp","10:boolean","11:boolean","13:tinyint","14:smallint","15:int","16:bigint","17:float","18:double","19:string","20:string","21:timestamp","22:timestamp","23:boolean","24:boolean"],"smallTableValueMapping:":["13:tinyint","14:smallint","16:bigint","17:float","18:double","19:string","20:string","21:timestamp","22:timestamp","23:boolean","24:boolean"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 46 Data size: 25096 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 46 Data size: 25096 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}}},"Map 2":{"Map Operator Tree:":[{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 2:int)"},"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_19","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_20","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col2","VALUE._col0":"_col0","VALUE._col1":"_col1","VALUE._col10":"_col11","VALUE._col2":"_col3","VALUE._col3":"_col4","VALUE._col4":"_col5","VALUE._col5":"_col6","VALUE._col6":"_col7","VALUE._col7":"_col8","VALUE._col8":"_col9","VALUE._col9":"_col10"},"key expressions:":"_col2 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col2 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["2:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:tinyint","1:smallint","3:bigint","4:float","5:double","6:string","7:string","8:timestamp","9:timestamp","10:boolean","11:boolean"]},"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)","OperatorId:":"RS_21","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_25"}}}}}} PREHOOK: query: select * from small_alltypesorc_b c left outer join small_alltypesorc_b cd @@ -346,6 +347,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 2","type":"BROADCAST_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_22","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 2":{"Map Operator Tree:":[{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_19","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_20","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_21","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_25"}}}}}} PREHOOK: query: select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b hd @@ -792,6 +794,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 3","type":"BROADCAST_EDGE"},{"parent":"Map 4","type":"BROADCAST_EDGE"}],"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_44","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 46 Data size: 148 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_45","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 920 Data size: 7360 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_46","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_47","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_48","outputname:":"Reducer 2","outputOperator:":["GBY_49"]}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 2:int)"},"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_38","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2]"},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_39","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["2:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_40","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_41","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_42","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_43","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_49","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_50"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_51"}}}}}} PREHOOK: query: select count(*) from (select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b cd diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out index 4a50a2fb89..c899f08265 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out @@ -86,6 +86,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 10390 Data size: 83120 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"st","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.*, st.* from sorted_mod_4 s left outer join small_table st @@ -125,6 +126,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cmodint` = 2 AS `=`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), (cmodint = 2) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"(cmodint = 2)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColEqualLongScalar(col 1:int, val 2) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 10390 Data size: 83120 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -164,6 +166,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, PMOD(CAST(`ctinyint` AS INTEGER), 4) = `cmodint` AS `=`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), ((UDFToInteger(ctinyint) pmod 4) = cmodint) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"((UDFToInteger(ctinyint) pmod 4) = cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4]","selectExpressions:":["LongColEqualLongColumn(col 3:int, col 1:int)(children: PosModLongToLong(col 0, divisor 4)(children: col 0:tinyint) -> 3:int) -> 4:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 4:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 10390 Data size: 83120 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -203,6 +206,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `ctinyint` < 100 AS `<`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`<`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), (ctinyint < 100Y) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"(ctinyint < 100Y)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColLessLongScalar(col 0:tinyint, val 100) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 10390 Data size: 83120 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -246,6 +250,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, CAST(`cmodint` AS BIGINT) AS `CAST`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `cbigint`\nFROM `default`.`small_table`\nWHERE PMOD(`cbigint`, 8) IS NOT NULL) AS `t1` ON PMOD(`t1`.`cbigint`, 8) = `t`.`CAST`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`sorted_mod_4`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 4","type":"BROADCAST_EDGE"},"Reducer 2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 5","type":"SIMPLE_EDGE"}],"Reducer 3":{"parent":"Reducer 2","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), UDFToLong(cmodint) (type: bigint)","columnExprMap:":{"_col0":"ctinyint","_col1":"UDFToLong(cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 6058 Data size: 72696 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_40","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint)","1":"(_col0 pmod 8L) (type: bigint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:bigint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 157508 Data size: 630032 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_41","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 157508 Data size: 630032 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_42","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_35"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["cbigint"],"database:":"default","filterExpr:":"(cbigint pmod 8L) is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 3:bigint)(children: PosModLongToLong(col 1, divisor 8) -> 3:bigint)"},"predicate:":"(cbigint pmod 8L) is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_37","children":{"Select Operator":{"expressions:":"cbigint (type: bigint)","columnExprMap:":{"_col0":"cbigint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[1]"},"Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_38","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"(_col0 pmod 8L)"},"key expressions:":"(_col0 pmod 8L) (type: bigint)","sort order:":"+","Map-reduce partition columns:":"(_col0 pmod 8L) (type: bigint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["3:bigint"],"keyExpressions:":["PosModLongToLong(col 1, divisor 8) -> 3:bigint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_39","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[1]","dataColumns:":["ctinyint:tinyint","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"s2","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_43","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_44","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_45","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_35"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Statistics:":"Num rows: 7339872 Data size: 58718976 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MERGEJOIN_35","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_16","outputname:":"Reducer 3","outputOperator:":["GBY_46"]}}}}}},"MergeJoin Vectorization:":{"enabled:":"false","enableConditionsNotMet:":["Vectorizing MergeJoin Supported IS false"]}},"Reducer 3":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_46","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_47"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_48"}}}}}} PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* from sorted_mod_4 s left outer join small_table sm @@ -355,6 +360,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 43163 Data size: 345304 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"st","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table2","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.*, st.* from mod_8_mod_4 s left outer join small_table2 st @@ -394,6 +400,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, `cmodint` = 2 AS `=`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), (cmodint = 2) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"(cmodint = 2)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColEqualLongScalar(col 1:int, val 2) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 43163 Data size: 345304 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table2","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -433,6 +440,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, PMOD(`cmodtinyint`, 4) = `cmodint` AS `=`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), ((cmodtinyint pmod 4) = cmodint) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"((cmodtinyint pmod 4) = cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4]","selectExpressions:":["LongColEqualLongColumn(col 3:int, col 1:int)(children: PosModLongToLong(col 0, divisor 4) -> 3:int) -> 4:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 4:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 43163 Data size: 345304 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table2","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -472,6 +480,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, `cmodtinyint` < 3 AS `<`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`<`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 3","type":"BROADCAST_EDGE"},"Reducer 2":{"parent":"Map 1","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), (cmodtinyint < 3) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"(cmodtinyint < 3)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColLessLongScalar(col 0:int, val 3) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 43163 Data size: 345304 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_28","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_29","outputname:":"Reducer 2","outputOperator:":["GBY_30"]}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table2","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_23","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_24","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 98 Data size: 388 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_25","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_30","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_31"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_32"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -515,6 +524,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, CAST(`cmodint` AS BIGINT) AS `CAST`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cbigint`\nFROM `default`.`small_table2`\nWHERE PMOD(`cbigint`, 8) IS NOT NULL) AS `t1` ON PMOD(`t1`.`cbigint`, 8) = `t`.`CAST`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`mod_8_mod_4`\nWHERE `cmodtinyint` IS NOT NULL) AS `t3` ON `t`.`cmodtinyint` = `t3`.`cmodtinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":{"parent":"Map 4","type":"BROADCAST_EDGE"},"Reducer 2":[{"parent":"Map 1","type":"SIMPLE_EDGE"},{"parent":"Map 5","type":"SIMPLE_EDGE"}],"Reducer 3":{"parent":"Reducer 2","type":"CUSTOM_SIMPLE_EDGE"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), UDFToLong(cmodint) (type: bigint)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"UDFToLong(cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 6058 Data size: 72696 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_40","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint)","1":"(_col0 pmod 8L) (type: bigint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:bigint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 157508 Data size: 630032 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_41","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 157508 Data size: 630032 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_42","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_35"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"sm","columns:":["cbigint"],"database:":"default","filterExpr:":"(cbigint pmod 8L) is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_table2","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cbigint:bigint, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 3:bigint)(children: PosModLongToLong(col 1, divisor 8) -> 3:bigint)"},"predicate:":"(cbigint pmod 8L) is not null (type: boolean)","Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_37","children":{"Select Operator":{"expressions:":"cbigint (type: bigint)","columnExprMap:":{"_col0":"cbigint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[1]"},"Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_38","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"(_col0 pmod 8L)"},"key expressions:":"(_col0 pmod 8L) (type: bigint)","sort order:":"+","Map-reduce partition columns:":"(_col0 pmod 8L) (type: bigint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["3:bigint"],"keyExpressions:":["PosModLongToLong(col 1, divisor 8) -> 3:bigint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 100 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_39","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[1]","dataColumns:":["cmodtinyint:int","cbigint:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}}},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"s2","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_43","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_44","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6058 Data size: 24232 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_45","outputname:":"Reducer 2","outputOperator:":["MERGEJOIN_35"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Execution mode:":"llap","Reduce Operator Tree:":{"Merge Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Statistics:":"Num rows: 119272933 Data size: 954183464 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MERGEJOIN_35","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.0","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_16","outputname:":"Reducer 3","outputOperator:":["GBY_46"]}}}}}},"MergeJoin Vectorization:":{"enabled:":"false","enableConditionsNotMet:":["Vectorizing MergeJoin Supported IS false"]}},"Reducer 3":{"Execution mode:":"vectorized, llap","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_46","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_47"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_48"}}}}}} PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* from mod_8_mod_4 s left outer join small_table2 sm diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out index c809983fcc..8869aefdce 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join6.q.out @@ -138,6 +138,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 2","type":"BROADCAST_EDGE"},{"parent":"Map 3","type":"BROADCAST_EDGE"}]},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_40","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:int"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"outerSmallTableKeyMapping:":["1 -> 5"],"projectedOutput:":["0:int","4:int","5:int"],"smallTableValueMapping:":["4:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2","_col3"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_41","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["5:int"],"bigTableRetainColumnNums:":"[0, 4]","bigTableValueColumns:":["0:int","4:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:int","4:int","6:int"],"smallTableValueMapping:":["6:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2","_col4"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_42","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4, 6]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_43","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_44"}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint]"}}},"Map 2":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 1:int)"},"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_34","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_35","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col1","VALUE._col0":"_col0"},"key expressions:":"_col1 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["1:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:int"]},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: int)","OperatorId:":"RS_36","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:char(2)"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 1:int)"},"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_37","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_38","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col1","VALUE._col0":"_col0"},"key expressions:":"_col1 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["1:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:int"]},"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: int)","OperatorId:":"RS_39","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:char(2)"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY @@ -172,6 +173,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Tez":{"DagId:":"###Masked###","Edges:":{"Map 1":[{"parent":"Map 2","type":"BROADCAST_EDGE"},{"parent":"Map 3","type":"BROADCAST_EDGE"}]},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_40","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:int"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"outerSmallTableKeyMapping:":["1 -> 5"],"projectedOutput:":["0:int","4:int","5:int"],"smallTableValueMapping:":["4:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2","_col3"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_41","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["5:int"],"bigTableRetainColumnNums:":"[0, 4]","bigTableValueColumns:":["0:int","4:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:int","4:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_42","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_43","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_44"}}}}}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}}},"Map 2":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 1:int)"},"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_34","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_35","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col1","VALUE._col0":"_col0"},"key expressions:":"_col1 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["1:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:int"]},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: int)","OperatorId:":"RS_36","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:char(2)"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Map 3":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:char(2), 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 1:int)"},"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_37","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[1]"},"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_38","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["1:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine tez IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"RS_39","outputname:":"Map 1","outputOperator:":["TS_0"]}}}}}}}}],"Execution mode:":"vectorized, llap","LLAP IO:":"all inputs","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[1]","dataColumns:":["rnum:int","c1:int","c2:char(2)"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/plan_json.q.out b/ql/src/test/results/clientpositive/plan_json.q.out index b16e574e3d..381eac2489 100644 --- a/ql/src/test/results/clientpositive/plan_json.q.out +++ b/ql/src/test/results/clientpositive/plan_json.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_7"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM `default`.`src`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_7"}}}}}} diff --git a/ql/src/test/results/clientpositive/spark/join0.q.out b/ql/src/test/results/clientpositive/spark/join0.q.out index 67850d8b60..a3ff14659a 100644 --- a/ql/src/test/results/clientpositive/spark/join0.q.out +++ b/ql/src/test/results/clientpositive/spark/join0.q.out @@ -120,6 +120,7 @@ SELECT src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +{"cboInfo":"Plan not optimized by CBO because the statement has sort by","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Spark":{"Edges:":{"Reducer 2":[{"parent":"Map 1","type":"PARTITION-LEVEL SORT","partitions":"1"},{"parent":"Map 4","type":"PARTITION-LEVEL SORT","partitions":"1"}],"Reducer 3":{"parent":"Reducer 2","type":"PARTITION-LEVEL SORT","partitions":"2"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","outputname:":"Reducer 2","outputOperator:":["JOIN_8"]}}}}}}}}]},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","outputname:":"Reducer 2","outputOperator:":["JOIN_8"]}}}}}}}}]},"Reducer 2":{"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","outputname:":"Reducer 3","outputOperator:":["SEL_11"]}}}}},"Reducer 3":{"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_15"}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Work 'Reducer 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/spark/parallel_join0.q.out b/ql/src/test/results/clientpositive/spark/parallel_join0.q.out index f830b6de33..b05c7fae8f 100644 --- a/ql/src/test/results/clientpositive/spark/parallel_join0.q.out +++ b/ql/src/test/results/clientpositive/spark/parallel_join0.q.out @@ -120,6 +120,7 @@ SELECT src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +{"cboInfo":"Plan not optimized by CBO because the statement has sort by","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Spark":{"Edges:":{"Reducer 2":[{"parent":"Map 1","type":"PARTITION-LEVEL SORT","partitions":"1"},{"parent":"Map 4","type":"PARTITION-LEVEL SORT","partitions":"1"}],"Reducer 3":{"parent":"Reducer 2","type":"PARTITION-LEVEL SORT","partitions":"4"}},"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","outputname:":"Reducer 2","outputOperator:":["JOIN_8"]}}}}}}}}]},"Map 4":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","outputname:":"Reducer 2","outputOperator:":["JOIN_8"]}}}}}}}}]},"Reducer 2":{"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","outputname:":"Reducer 3","outputOperator:":["SEL_11"]}}}}},"Reducer 3":{"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_15"}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Work 'Reducer 2' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out index ae67597400..49105d9d6e 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out @@ -248,7 +248,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[6]","bigTableValueColumns:":["6:string"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col1"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 22 Data size: 4770 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_31","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 24 Data size: 5247 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_32","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_33","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_34","outputname:":"Reducer 2","outputOperator:":["GBY_35"]}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_35","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_36"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -292,7 +292,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["7:string"],"bigTableRetainColumnNums:":"[6]","bigTableValueColumns:":["6:string"],"className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 22 Data size: 4770 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_31","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterStringOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 24 Data size: 5247 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_32","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_33","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_34","outputname:":"Reducer 2","outputOperator:":["GBY_35"]}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_35","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_36"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -336,7 +336,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 4337 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["3:bigint","7:string"],"bigTableRetainColumnNums:":"[2, 6]","bigTableValueColumns:":["2:int","6:string"],"className:":"VectorMapJoinOuterMultiKeyOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["2:int","6:string"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col2"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 22 Data size: 4770 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_31","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int","6:string"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterMultiKeyOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 24 Data size: 5247 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_32","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_33","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_34","outputname:":"Reducer 2","outputOperator:":["GBY_35"]}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_35","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_36"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out index f63af39bea..2ad34a5368 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out @@ -262,7 +262,7 @@ left outer join small_alltypesorc_b cd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_15","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","bigTableValueColumns:":["0:tinyint","1:smallint","2:int","3:bigint","4:float","5:double","6:string","7:string","8:timestamp","9:timestamp","10:boolean","11:boolean"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"outerSmallTableKeyMapping:":["2 -> 15"],"projectedOutput:":["0:tinyint","1:smallint","2:int","3:bigint","4:float","5:double","6:string","7:string","8:timestamp","9:timestamp","10:boolean","11:boolean","13:tinyint","14:smallint","15:int","16:bigint","17:float","18:double","19:string","20:string","21:timestamp","22:timestamp","23:boolean","24:boolean"],"smallTableValueMapping:":["13:tinyint","14:smallint","16:bigint","17:float","18:double","19:string","20:string","21:timestamp","22:timestamp","23:boolean","24:boolean"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 33 Data size: 7686 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_16","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 33 Data size: 7686 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_22"}}}}}} PREHOOK: query: select * from small_alltypesorc_b c left outer join small_alltypesorc_b cd @@ -347,7 +347,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_15","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 2"},"Statistics:":"Num rows: 33 Data size: 7686 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_16","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 33 Data size: 7686 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_17"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_22"}}}}}} PREHOOK: query: select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b hd @@ -794,7 +794,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 6988 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["2:int"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 33 Data size: 7686 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_31","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 36 Data size: 8454 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_32","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_33","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_34","outputname:":"Reducer 2","outputOperator:":["GBY_35"]}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_35","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_36"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_45"}}}}}} PREHOOK: query: select count(*) from (select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b cd diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out index 98732edac8..42e8a63ff5 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out @@ -86,7 +86,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.*, st.* from sorted_mod_4 s left outer join small_table st @@ -126,7 +126,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cmodint` = 2 AS `=`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), (cmodint = 2) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"(cmodint = 2)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColEqualLongScalar(col 1:int, val 2) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -166,7 +166,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, PMOD(CAST(`ctinyint` AS INTEGER), 4) = `cmodint` AS `=`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), ((UDFToInteger(ctinyint) pmod 4) = cmodint) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"((UDFToInteger(ctinyint) pmod 4) = cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4]","selectExpressions:":["LongColEqualLongColumn(col 3:int, col 1:int)(children: PosModLongToLong(col 0, divisor 4)(children: col 0:tinyint) -> 3:int) -> 4:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 4:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -206,7 +206,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `ctinyint` < 100 AS `<`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_table`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint` AND `t`.`<`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), (ctinyint < 100Y) (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"(ctinyint < 100Y)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColLessLongScalar(col 0:tinyint, val 100) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:tinyint"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint from sorted_mod_4 s left outer join small_table sm @@ -250,7 +250,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table POSTHOOK: Input: default@sorted_mod_4 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, CAST(`cmodint` AS BIGINT) AS `CAST`\nFROM `default`.`sorted_mod_4`) AS `t`\nLEFT JOIN (SELECT `cbigint`\nFROM `default`.`small_table`\nWHERE PMOD(`cbigint`, 8) IS NOT NULL) AS `t1` ON PMOD(`t1`.`cbigint`, 8) = `t`.`CAST`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`sorted_mod_4`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["ctinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), UDFToLong(cmodint) (type: bigint)","columnExprMap:":{"_col0":"ctinyint","_col1":"UDFToLong(cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint)","1":"(_col0 pmod 8L) (type: bigint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:bigint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:tinyint"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:tinyint"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_27","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_28","outputname:":"Reducer 2","outputOperator:":["JOIN_13"]}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"s2","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"sorted_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:tinyint)"},"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_29","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: tinyint)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: tinyint)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:tinyint"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_31","outputname:":"Reducer 2","outputOperator:":["JOIN_13"]}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"notVectorizedReason:":"Tagging not supported","vectorized:":"false"},"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Statistics:":"Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_13","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_16","outputname:":"Reducer 3","outputOperator:":["GBY_32"]}}}}}}},"Reducer 3":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_32","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_33"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_38"}}}}}} PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* from sorted_mod_4 s left outer join small_table sm @@ -360,7 +360,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.*, st.* from mod_8_mod_4 s left outer join small_table2 st @@ -400,7 +400,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, `cmodint` = 2 AS `=`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), (cmodint = 2) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"(cmodint = 2)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColEqualLongScalar(col 1:int, val 2) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -440,7 +440,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, PMOD(`cmodtinyint`, 4) = `cmodint` AS `=`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`=`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), ((cmodtinyint pmod 4) = cmodint) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"((cmodtinyint pmod 4) = cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 4]","selectExpressions:":["LongColEqualLongColumn(col 3:int, col 1:int)(children: PosModLongToLong(col 0, divisor 4) -> 3:int) -> 4:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 4:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -480,7 +480,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, `cmodtinyint` < 3 AS `<`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`small_table2`\nWHERE `cmodtinyint` IS NOT NULL) AS `t1` ON `t`.`cmodtinyint` = `t1`.`cmodtinyint` AND `t`.`<`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), (cmodtinyint < 3) (type: boolean)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"(cmodtinyint < 3)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 3]","selectExpressions:":["LongColLessLongScalar(col 0:int, val 3) -> 3:boolean"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_19","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"filter predicates:":{"0":"{_col1}","1":""},"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableFilterExpressions:":["SelectColumnIsTrue(col 3:boolean)"],"bigTableKeyColumns:":["0:int"],"bigTableRetainColumnNums:":"[]","className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"hashTableImplementationType:":"OPTIMIZED"},"input vertices:":{"1":"Map 3"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_20","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_21","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkEmptyKeyOperator","native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"valueColumns:":["0:bigint"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_22","outputname:":"Reducer 2","outputOperator:":["GBY_23"]}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Reducer 2":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_23","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_24"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_29"}}}}}} PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint from mod_8_mod_4 s left outer join small_table2 sm @@ -524,7 +524,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 POSTHOOK: Input: default@small_table2 POSTHOOK: Output: hdfs://### HDFS PATH ### -#### A masked pattern was here #### +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cmodtinyint`, CAST(`cmodint` AS BIGINT) AS `CAST`\nFROM `default`.`mod_8_mod_4`) AS `t`\nLEFT JOIN (SELECT `cbigint`\nFROM `default`.`small_table2`\nWHERE PMOD(`cbigint`, 8) IS NOT NULL) AS `t1` ON PMOD(`t1`.`cbigint`, 8) = `t`.`CAST`\nLEFT JOIN (SELECT `cmodtinyint`\nFROM `default`.`mod_8_mod_4`\nWHERE `cmodtinyint` IS NOT NULL) AS `t3` ON `t`.`cmodtinyint` = `t3`.`cmodtinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-2":{"ROOT STAGE":"TRUE"},"Stage-1":{"DEPENDENT STAGES":"Stage-2"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-2":{"Spark":{"DagName:":"###Masked###","Vertices:":{"Map 1":{"Map Operator Tree:":[{"TableScan":{"alias:":"s","columns:":["cmodtinyint","cmodint"],"database:":"default","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int), UDFToLong(cmodint) (type: bigint)","columnExprMap:":{"_col0":"cmodtinyint","_col1":"UDFToLong(cmodint)"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint)","1":"(_col0 pmod 8L) (type: bigint)"},"Map Join Vectorization:":{"bigTableKeyColumns:":["1:bigint"],"bigTableRetainColumnNums:":"[0]","bigTableValueColumns:":["0:int"],"className:":"VectorMapJoinOuterLongOperator","native:":"true","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"projectedOutput:":["0:int"],"hashTableImplementationType:":"OPTIMIZED"},"outputColumnNames:":["_col0"],"input vertices:":{"1":"Map 4"},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"MAPJOIN_27","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_28","outputname:":"Reducer 2","outputOperator:":["JOIN_13"]}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0, 1]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}},"Map 5":{"Map Operator Tree:":[{"TableScan":{"alias:":"s2","columns:":["cmodtinyint"],"database:":"default","filterExpr:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","table:":"mod_8_mod_4","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:cmodtinyint:int, 1:cmodint:int, 2:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"Filter Vectorization:":{"className:":"VectorFilterOperator","native:":"true","predicateExpression:":"SelectColumnIsNotNull(col 0:int)"},"predicate:":"cmodtinyint is not null (type: boolean)","Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_29","children":{"Select Operator":{"expressions:":"cmodtinyint (type: int)","columnExprMap:":{"_col0":"cmodtinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_30","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0"},"key expressions:":"_col0 (type: int)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: int)","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkLongOperator","keyColumns:":["0:int"],"native:":"true","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"]},"Statistics:":"Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_31","outputname:":"Reducer 2","outputOperator:":["JOIN_13"]}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"true","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"2","includeColumns:":"[0]","dataColumns:":["cmodtinyint:int","cmodint:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}}},"Reducer 2":{"Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"notVectorizedReason:":"Tagging not supported","vectorized:":"false"},"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Statistics:":"Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_13","children":{"Group By Operator":{"aggregations:":["count()"],"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_16","outputname:":"Reducer 3","outputOperator:":["GBY_32"]}}}}}}},"Reducer 3":{"Execution mode:":"vectorized","Reduce Vectorization:":{"enabled:":"true","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true","hive.execution.engine spark IN [tez, spark] IS true"],"reduceColumnNullOrder:":"","reduceColumnSortOrder:":"","allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"1","dataColumns:":["VALUE._col0:bigint"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountMerge(col 0:bigint) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"MERGEPARTIAL","native:":"false","vectorProcessingMode:":"GLOBAL","projectedOutputColumnNums:":"[0]"},"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"GBY_32","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_33"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_38"}}}}}} PREHOOK: query: select count(*) from (select s.*, sm.*, s2.* from mod_8_mod_4 s left outer join small_table2 sm diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out index 57a4f0cf48..7ee5d4cbb5 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -248,7 +248,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 45 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 101 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 6]"},"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 45 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 101 Data size: 808 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 6]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -292,7 +292,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 23 Data size: 1260 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2`\nLEFT JOIN (SELECT `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cstring2"],"database:":"default","filterExpr:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 1488 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cstring2 is not null (type: boolean)","Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cstring2 (type: string)","columnExprMap:":{"_col0":"cstring2"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 15 Data size: 1116 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","filterExpr:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 990 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"cstring1 is not null (type: boolean)","Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cstring1 (type: string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 10 Data size: 540 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cstring1","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[6, 7]"},"Statistics:":"Num rows: 20 Data size: 2478 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 7:string"],"bigTableValueExpressions:":["col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 23 Data size: 1260 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 51 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd @@ -336,7 +336,7 @@ left outer join small_alltypesorc_a_n1 hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a_n1 #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cstring2 is not null and cbigint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cstring1 is not null and cint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `cint`, `cbigint`, `cstring1`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`) AS `t`\nLEFT JOIN (SELECT `cbigint`, `cstring2`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring2` IS NOT NULL AND `cbigint` IS NOT NULL) AS `t1` ON `t`.`cstring2` = `t1`.`cstring2` AND `t`.`cbigint` = `t1`.`cbigint`\nLEFT JOIN (SELECT `cint`, `cstring1`\nFROM `default`.`small_alltypesorc_a_n1`\nWHERE `cstring1` IS NOT NULL AND `cint` IS NOT NULL) AS `t3` ON `t`.`cstring1` = `t3`.`cstring1` AND `t`.`cint` = `t3`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cbigint","cstring2"],"database:":"default","filterExpr:":"(cstring2 is not null and cbigint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"(cbigint is not null and cstring2 is not null) (type: boolean)","Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cbigint (type: bigint), cstring2 (type: string)","columnExprMap:":{"_col0":"cbigint","_col1":"cstring2"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 11 Data size: 909 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cint","cstring1"],"database:":"default","filterExpr:":"(cstring1 is not null and cint is not null) (type: boolean)","Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"(cint is not null and cstring1 is not null) (type: boolean)","Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"cint (type: int), cstring1 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 5 Data size: 282 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cbigint","cstring1","cstring2"],"database:":"default","Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_a_n1","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)","columnExprMap:":{"_col0":"cint","_col1":"cbigint","_col2":"cstring1","_col3":"cstring2"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2, 3, 6, 7]"},"Statistics:":"Num rows: 20 Data size: 2650 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: bigint), _col3 (type: string)","1":"_col0 (type: bigint), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 3:bigint","col 7:string"],"bigTableValueExpressions:":["col 2:int","col 6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 20 Data size: 1034 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int), _col2 (type: string)","1":"_col0 (type: int), _col1 (type: string)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:int","col 1:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 32 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[2, 3, 6, 7]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.cstring1 from small_alltypesorc_a_n1 c left outer join small_alltypesorc_a_n1 cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out index 1cfd6f7a90..060e7b4cf6 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -262,7 +262,7 @@ left outer join small_alltypesorc_b cd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT *\nFROM (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`, `csmallint`, `cint`, `cbigint`, `cfloat`, `cdouble`, `cstring1`, `cstring2`, `ctimestamp1`, `ctimestamp2`, `cboolean1`, `cboolean2`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Statistics:":"Num rows: 20 Data size: 4864 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"],"database:":"default","Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)","columnExprMap:":{"_col0":"ctinyint","_col1":"csmallint","_col10":"cboolean1","_col11":"cboolean2","_col2":"cint","_col3":"cbigint","_col4":"cfloat","_col5":"cdouble","_col6":"cstring1","_col7":"cstring2","_col8":"ctimestamp1","_col9":"ctimestamp2"},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"},"Statistics:":"Num rows: 30 Data size: 7296 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col10":"0:_col10","_col11":"0:_col11","_col12":"1:_col0","_col13":"1:_col1","_col14":"1:_col2","_col15":"1:_col3","_col16":"1:_col4","_col17":"1:_col5","_col18":"1:_col6","_col19":"1:_col7","_col2":"0:_col2","_col20":"1:_col8","_col21":"1:_col9","_col22":"1:_col10","_col23":"1:_col11","_col3":"0:_col3","_col4":"0:_col4","_col5":"0:_col5","_col6":"0:_col6","_col7":"0:_col7","_col8":"0:_col8","_col9":"0:_col9"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col2 (type: int)","1":"_col2 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint","col 1:smallint","col 2:int","col 3:bigint","col 4:float","col 5:double","col 6:string","col 7:string","col 8:timestamp","col 9:timestamp","col 10:boolean","col 11:boolean"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"],"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 76 Data size: 34428 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint, bigint, bigint, double, double, string, string, timestamp, timestamp, bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select * from small_alltypesorc_b c left outer join small_alltypesorc_b cd @@ -347,7 +347,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`ctinyint`\nFROM (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t1` ON `t`.`ctinyint` = `t1`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-4":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-4"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-4":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_10","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_12"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0]"},"Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_14","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_15","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 600 Data size: 2364 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} PREHOOK: query: select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b hd @@ -794,7 +794,7 @@ left outer join small_alltypesorc_b hd POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b #### A masked pattern was here #### -{"optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 58 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1160 Data size: 9280 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT COUNT(*) AS `$f0`\nFROM (SELECT `ctinyint`, `cint`\nFROM `default`.`small_alltypesorc_b`) AS `t`\nLEFT JOIN (SELECT `cint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `cint` IS NOT NULL) AS `t1` ON `t`.`cint` = `t1`.`cint`\nLEFT JOIN (SELECT `ctinyint`\nFROM `default`.`small_alltypesorc_b`\nWHERE `ctinyint` IS NOT NULL) AS `t3` ON `t`.`ctinyint` = `t3`.`ctinyint`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-8":{"ROOT STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:cd":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","filterExpr:":"cint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"cint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_20","children":{"Select Operator":{"expressions:":"cint (type: int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_30"}}}}}}}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["ctinyint"],"database:":"default","filterExpr:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 30 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"ctinyint is not null (type: boolean)","Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_21","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint)","columnExprMap:":{"_col0":"ctinyint"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 20 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"OperatorId:":"HASHTABLESINK_28"}}}}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"c","columns:":["ctinyint","cint"],"database:":"default","Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","table:":"small_alltypesorc_b","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"ctinyint (type: tinyint), cint (type: int)","columnExprMap:":{"_col0":"ctinyint","_col1":"cint"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 2]"},"Statistics:":"Num rows: 30 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_32","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 58 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_33","children":{"Map Join Operator":{"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: tinyint)","1":"_col0 (type: tinyint)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 0:tinyint"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1160 Data size: 9280 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_34","children":{"Group By Operator":{"aggregations:":["count()"],"Group By Vectorization:":{"aggregators:":["VectorUDAFCountStar(*) -> bigint"],"className:":"VectorGroupByOperator","groupByMode:":"HASH","native:":"false","vectorProcessingMode:":"HASH","projectedOutputColumnNums:":"[0]"},"minReductionHashAggr:":"0.99","mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_35","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0"},"sort order:":"","Reduce Sink Vectorization:":{"className:":"VectorReduceSinkOperator","native:":"false","nativeConditionsMet:":["hive.vectorized.execution.reducesink.new.enabled IS true","No PTF TopN IS true","No DISTINCT columns IS true","BinarySortableSerDe for keys IS true","LazyBinarySerDe for values IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_36"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"12","includeColumns:":"[0, 2]","dataColumns:":["ctinyint:tinyint","csmallint:smallint","cint:int","cbigint:bigint","cfloat:float","cdouble:double","cstring1:string","cstring2:string","ctimestamp1:timestamp","ctimestamp2:timestamp","cboolean1:boolean","cboolean2:boolean"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[]"}},"Local Work:":{"Map Reduce Local Work":{}},"Reduce Vectorization:":{"enabled:":"false","enableConditionsMet:":["hive.vectorized.execution.reduce.enabled IS true"],"enableConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_17","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_19"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_37"}}}}}} PREHOOK: query: select count(*) from (select c.ctinyint from small_alltypesorc_b c left outer join small_alltypesorc_b cd diff --git a/ql/src/test/results/clientpositive/vector_outer_join6.q.out b/ql/src/test/results/clientpositive/vector_outer_join6.q.out index 7269177c4b..06d7ef8fdc 100644 --- a/ql/src/test/results/clientpositive/vector_outer_join6.q.out +++ b/ql/src/test/results/clientpositive/vector_outer_join6.q.out @@ -138,7 +138,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `tj2rnum`, `t3`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2","_col4":"1:_col0"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col4"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int), _col4 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2","_col2":"_col4"},"outputColumnNames:":["_col0","_col1","_col2"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1, 2]"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY @@ -173,7 +173,7 @@ POSTHOOK: Input: default@tjoin1_n0 POSTHOOK: Input: default@tjoin2_n0 POSTHOOK: Input: default@tjoin3 #### A masked pattern was here #### -{"optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} +{"CBOPlan":"###Masked###","optimizedSQL":"SELECT `t`.`rnum` AS `tj1rnum`, `t1`.`rnum` AS `rnumt3`\nFROM (SELECT `rnum`, `c1`\nFROM `default`.`tjoin1_n0`) AS `t`\nLEFT JOIN (SELECT `rnum`, `c1`\nFROM `default`.`tjoin2_n0`\nWHERE `c1` IS NOT NULL) AS `t1` ON `t`.`c1` = `t1`.`c1`\nLEFT JOIN (SELECT `c1`\nFROM `default`.`tjoin3`\nWHERE `c1` IS NOT NULL) AS `t3` ON `t1`.`c1` = `t3`.`c1`","PLAN VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled IS true"]},"cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias -> Map Local Tables:":{"$hdt$_1:tjoin2_n0":{"Fetch Operator":{"limit:":"-1"}},"$hdt$_2:tjoin3":{"Fetch Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator Tree:":{"$hdt$_1:tjoin2_n0":{"TableScan":{"alias:":"tjoin2_n0","columns:":["rnum","c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin2_n0","isTempTable:":"false","OperatorId:":"TS_2","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_16","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_4","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"OperatorId:":"HASHTABLESINK_24"}}}}}}}},"$hdt$_2:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["c1"],"database:":"default","filterExpr:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_5","children":{"Filter Operator":{"predicate:":"c1 is not null (type: boolean)","Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"c1 (type: int)","columnExprMap:":{"_col0":"c1"},"outputColumnNames:":["_col0"],"Statistics:":"Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_7","children":{"HashTable Sink Operator":{"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"OperatorId:":"HASHTABLESINK_22"}}}}}}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"tjoin1_n0","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","table:":"tjoin1_n0","TableScan Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 1:c1:int, 2:c2:int, 3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"rnum (type: int), c1 (type: int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_26","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"1:_col0","_col3":"1:_col1"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: int)","1":"_col1 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 1:int"],"bigTableValueExpressions:":["col 0:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2","_col3"],"Statistics:":"Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_27","children":{"Map Join Operator":{"columnExprMap:":{"_col0":"0:_col0","_col2":"0:_col2"},"condition map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col3 (type: int)","1":"_col0 (type: int)"},"Map Join Vectorization:":{"bigTableKeyExpressions:":["col 2:int"],"bigTableValueExpressions:":["col 0:int","col 1:int"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS true","Outer Join has keys IS true","Optimized Table and Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS false"]},"outputColumnNames:":["_col0","_col2"],"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"MAPJOIN_28","children":{"Select Operator":{"expressions:":"_col0 (type: int), _col2 (type: int)","columnExprMap:":{"_col0":"_col0","_col1":"_col2"},"outputColumnNames:":["_col0","_col1"],"Select Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0, 1]"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_29","children":{"File Output Operator":{"compressed:":"false","File Sink Vectorization:":{"className:":"VectorFileSinkOperator","native:":"false"},"Statistics:":"Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_30"}}}}}}}}}}}}],"Execution mode:":"vectorized","Map Vectorization:":{"enabled:":"true","enabledConditionsMet:":["hive.vectorized.use.vectorized.input.format IS true"],"inputFormatFeatureSupport:":"[DECIMAL_64]","featureSupportInUse:":"[DECIMAL_64]","inputFileFormats:":["org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"],"allNative:":"false","usesVectorUDFAdaptor:":"false","vectorized:":"true","rowBatchContext:":{"dataColumnCount:":"3","includeColumns:":"[0, 1]","dataColumns:":["rnum:int","c1:int","c2:int"],"partitionColumnCount:":"0","scratchColumnTypeNames:":"[bigint, bigint]"}},"Local Work:":{"Map Reduce Local Work":{}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_31"}}}}}} PREHOOK: query: select tj1rnum, tj2rnum as rnumt3 from (select tjoin1_n0.rnum tj1rnum, tjoin2_n0.rnum tj2rnum, tjoin2_n0.c1 tj2c1 from tjoin1_n0 left outer join tjoin2_n0 on tjoin1_n0.c1 = tjoin2_n0.c1 ) tj left outer join tjoin3 on tj2c1 = tjoin3.c1 PREHOOK: type: QUERY