diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fb07c7e..3907340 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -641,7 +641,7 @@ "How many rows in the joining tables (except the streaming table) should be cached in memory."), // CBO related - HIVE_CBO_ENABLED("hive.cbo.enable", true, "Flag to control enabling Cost Based Optimizations using Optiq framework."), + HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Optiq framework."), // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) diff --git data/conf/hive-site.xml data/conf/hive-site.xml index 50027de..ed3ee42 100644 --- data/conf/hive-site.xml +++ data/conf/hive-site.xml @@ -25,12 +25,6 @@ Internal marker for test. Used for masking env-dependent values - - hive.cbo.enable - true - Enable CBO for Unit Tests - - diff --git ql/src/test/results/clientpositive/tez/cbo_correctness.q.out ql/src/test/results/clientpositive/tez/cbo_correctness.q.out index dabc888..5920612 100644 --- ql/src/test/results/clientpositive/tez/cbo_correctness.q.out +++ ql/src/test/results/clientpositive/tez/cbo_correctness.q.out @@ -16734,26 +16734,6 @@ POSTHOOK: Input: default@t1@dt=2014 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t2@dt=2014 #### A masked pattern was here #### - 1 1 1 1.0 true 2014 - 1 1 1 1.0 true 2014 - 1 1 1 1.0 true 2014 - 1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 false 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 false 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 -1 1 1 1.0 true 2014 -2 2 2 2.0 true 2014 -2 2 2 2.0 true 2014 -2 2 2 2.0 true 2014 -2 2 2 2.0 true 2014 -2 2 2 2.0 true 2014 -null null NULL NULL NULL 2014 -null null NULL NULL NULL 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 1 1 1 1.0 true 2014 @@ -16774,6 +16754,26 @@ null null NULL NULL NULL 2014 1 1 1 1.0 false 2014 null null NULL NULL NULL 2014 null null NULL NULL NULL 2014 + 1 1 1 1.0 true 2014 + 1 1 1 1.0 true 2014 + 1 1 1 1.0 true 2014 + 1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 false 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 false 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +1 1 1 1.0 true 2014 +2 2 2 2.0 true 2014 +2 2 2 2.0 true 2014 +2 2 2 2.0 true 2014 +2 2 2 2.0 true 2014 +2 2 2 2.0 true 2014 +null null NULL NULL NULL 2014 +null null NULL NULL NULL 2014 PREHOOK: query: select key from (select key, c_int from (select * from t1 union all select * from t2 where t2.key >=0)r1 union all select key, c_int from t3)r2 where key >=0 order by key PREHOOK: type: QUERY PREHOOK: Input: default@t1 diff --git ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out index 6b63ab9..3f19f95 100644 --- ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out +++ ql/src/test/results/clientpositive/tez/correlationoptimizer1.q.out @@ -352,6 +352,8 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) outputColumnNames: _col0 + input vertices: + 0 Map 1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out index 78aeff0..8467df2 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out @@ -3521,6 +3521,8 @@ STAGE PLANS: keys: 0 ds (type: string) 1 ds (type: string) + input vertices: + 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -3654,6 +3656,8 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) outputColumnNames: _col3 + input vertices: + 1 Map 4 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -3664,6 +3668,8 @@ STAGE PLANS: keys: 0 _col3 (type: string) 1 hr (type: string) + input vertices: + 1 Map 3 Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2420 Data size: 25709 Basic stats: COMPLETE Column stats: NONE @@ -3825,6 +3831,8 @@ STAGE PLANS: keys: 0 ds (type: string), hr (type: string) 1 ds (type: string), hr (type: string) + input vertices: + 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -3969,6 +3977,8 @@ STAGE PLANS: keys: 0 ds (type: string) 1 ds (type: string) + input vertices: + 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4077,6 +4087,8 @@ STAGE PLANS: keys: 0 UDFToDouble(hr) (type: double) 1 UDFToDouble(UDFToInteger((hr / 2))) (type: double) + input vertices: + 1 Map 3 Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -4196,6 +4208,8 @@ STAGE PLANS: keys: 0 (hr * 2) (type: double) 1 hr (type: double) + input vertices: + 1 Map 3 Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -4378,6 +4392,8 @@ STAGE PLANS: keys: 0 ds (type: string) 1 _col0 (type: string) + input vertices: + 0 Map 1 Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -4478,6 +4494,8 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) outputColumnNames: _col8 + input vertices: + 1 Map 3 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col8 = '2008-04-08') (type: boolean) @@ -4573,6 +4591,8 @@ STAGE PLANS: keys: 0 ds (type: string) 1 ds (type: string) + input vertices: + 1 Map 1 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE @@ -4734,6 +4754,8 @@ STAGE PLANS: 0 ds (type: string) 1 ds (type: string) outputColumnNames: _col3 + input vertices: + 1 Map 4 Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -4744,6 +4766,8 @@ STAGE PLANS: keys: 0 _col3 (type: string) 1 '11' (type: string) + input vertices: + 1 Map 3 Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1210 Data size: 12854 Basic stats: COMPLETE Column stats: NONE @@ -4990,6 +5014,8 @@ STAGE PLANS: 0 ds (type: string) 1 _col0 (type: string) outputColumnNames: _col2 + input vertices: + 1 Union 3 Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col2 (type: string) @@ -5207,6 +5233,8 @@ STAGE PLANS: keys: 0 ds (type: string), UDFToDouble(hr) (type: double) 1 ds (type: string), UDFToDouble(hr) (type: double) + input vertices: + 1 Map 3 Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE Select Operator Statistics: Num rows: 1100 Data size: 0 Basic stats: PARTIAL Column stats: NONE diff --git ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out index 80967ec..cf56f07 100644 --- ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out +++ ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out @@ -178,6 +178,8 @@ STAGE PLANS: 0 dim_shops_id (type: int) 1 id (type: int) outputColumnNames: _col0, _col1, _col5, _col6 + input vertices: + 1 Map 1 Statistics: Num rows: 0 Data size: 39 Basic stats: PARTIAL Column stats: NONE Filter Operator predicate: ((_col1 = _col5) and (_col6) IN ('foo', 'bar')) (type: boolean) @@ -362,6 +364,8 @@ STAGE PLANS: 0 dim_shops_id (type: int) 1 id (type: int) outputColumnNames: _col0, _col1, _col5 + input vertices: + 1 Map 1 Filter Operator predicate: (_col1 = _col5) (type: boolean) Select Operator @@ -391,6 +395,8 @@ STAGE PLANS: 0 dim_shops_id (type: int) 1 id (type: int) outputColumnNames: _col0, _col1, _col5 + input vertices: + 1 Map 2 Filter Operator predicate: (_col1 = _col5) (type: boolean) Select Operator diff --git ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out index 72b4c57..2a3e56c 100644 --- ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/mapjoin_mapjoin.q.out @@ -70,67 +70,63 @@ STAGE PLANS: GatherStats: false Filter Operator isSamplingPred: false - predicate: (key is not null and value is not null) (type: boolean) + predicate: (value is not null and key is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {key} {value} + 0 {key} 1 - Estimated key counts: Map 3 => 13 + Estimated key counts: Map 3 => 250 keys: - 0 key (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0 input vertices: 1 Map 3 Position of Big Table: 0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {_col0} - Estimated key counts: Map 2 => 250 - keys: - 0 value (type: string) - 1 _col1 (type: string) - outputColumnNames: _col5 - input vertices: - 0 Map 2 - Position of Big Table: 1 + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {_col0} + 1 + Estimated key counts: Map 2 => 13 + keys: + 0 _col0 (type: string) + 1 key (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 2 + Position of Big Table: 0 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col5 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 0 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0 - columns.types string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -326,26 +322,26 @@ STAGE PLANS: Map 2 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: value is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: value (type: string) + key expressions: key (type: string) sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - tag: 0 + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + tag: 1 auto parallelism: true Path -> Alias: #### A masked pattern was here #### Path -> Partition: #### A masked pattern was here #### Partition - base file name: src + base file name: src1 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -355,14 +351,14 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -375,35 +371,35 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.src + name default.src1 numFiles 1 - numRows 500 - rawDataSize 5312 - serialization.ddl struct src { string key, string value} + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 5812 + totalSize 216 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src - name: default.src + name: default.src1 + name: default.src1 Truncated Path -> Alias: - /src [src] + /src1 [src1] Map 3 Map Operator Tree: TableScan - alias: src1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: key (type: string) + key expressions: value (type: string) sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true Path -> Alias: @@ -411,7 +407,7 @@ STAGE PLANS: Path -> Partition: #### A masked pattern was here #### Partition - base file name: src1 + base file name: src input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: @@ -421,14 +417,14 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -441,20 +437,20 @@ STAGE PLANS: columns.comments defaultdefault columns.types string:string #### A masked pattern was here #### - name default.src1 + name default.src numFiles 1 - numRows 25 - rawDataSize 191 - serialization.ddl struct src1 { string key, string value} + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - totalSize 216 + totalSize 5812 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.src1 - name: default.src1 + name: default.src + name: default.src Truncated Path -> Alias: - /src1 [src1] + /src [src] Stage: Stage-0 Fetch Operator @@ -476,80 +472,55 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) + Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((value > 'val_450') and value is not null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Map 2 - Map Operator Tree: - TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((value > 'val_450') and key is not null) and value is not null) (type: boolean) - Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE + predicate: ((value is not null and key is not null) and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: _col0 + input vertices: + 1 Map 3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 {_col0} {_col1} + 0 {_col0} 1 keys: 0 _col0 (type: string) 1 key (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: _col0 input vertices: - 1 Map 3 - Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE + 1 Map 2 + Statistics: Num rows: 200 Data size: 2132 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 - 1 {_col0} - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col5 - input vertices: - 0 Map 1 - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col5 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 201 Data size: 2146 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 3 + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 200 Data size: 2132 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 200 Data size: 2132 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 2 Map Operator Tree: TableScan alias: src1 @@ -562,6 +533,19 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and (value > 'val_450')) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -583,39 +567,13 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: src1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 2 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: value is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: value (type: string) - sort order: + - Map-reduce partition columns: value (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -632,7 +590,7 @@ STAGE PLANS: 1 value (type: string) outputColumnNames: _col0, _col2 input vertices: - 1 Map 2 + 1 Map 4 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -645,15 +603,15 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col2 input vertices: - 1 Map 1 + 1 Map 3 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string) - outputColumnNames: _col0 + outputColumnNames: _col2 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() - keys: _col0 (type: string) + keys: _col2 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE @@ -663,7 +621,33 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 4 + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reducer 2 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) diff --git ql/src/test/results/clientpositive/tez/metadataonly1.q.out ql/src/test/results/clientpositive/tez/metadataonly1.q.out index 77d72cf..fa22920 100644 --- ql/src/test/results/clientpositive/tez/metadataonly1.q.out +++ ql/src/test/results/clientpositive/tez/metadataonly1.q.out @@ -1222,7 +1222,7 @@ POSTHOOK: Input: default@test2@ds=1/hr=1 POSTHOOK: Input: default@test2@ds=1/hr=2 POSTHOOK: Input: default@test2@ds=1/hr=3 #### A masked pattern was here #### -1 1 +1 3 PREHOOK: query: explain extended select ds, count(hr) from TEST2 group by ds PREHOOK: type: QUERY POSTHOOK: query: explain extended select ds, count(hr) from TEST2 group by ds @@ -1684,6 +1684,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### 2008-04-08 +2008-04-09 PREHOOK: query: select min(ds),max(ds) from srcpart PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -1700,7 +1701,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-08 +2008-04-08 2008-04-09 PREHOOK: query: -- HIVE-3594 URI encoding for temporary path alter table TEST2 add partition (ds='01:10:10', hr='01') PREHOOK: type: ALTERTABLE_ADDPARTS @@ -2058,3 +2059,5 @@ POSTHOOK: Input: default@test2@ds=1/hr=2 POSTHOOK: Input: default@test2@ds=1/hr=3 #### A masked pattern was here #### 01:10:10 1 +01:10:20 1 +1 3 diff --git ql/src/test/results/clientpositive/tez/mrr.q.out ql/src/test/results/clientpositive/tez/mrr.q.out index 6bbbda3..0507c1e 100644 --- ql/src/test/results/clientpositive/tez/mrr.q.out +++ ql/src/test/results/clientpositive/tez/mrr.q.out @@ -1281,15 +1281,14 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Reducer 11 <- Map 10 (SIMPLE_EDGE) - Reducer 12 <- Reducer 11 (SIMPLE_EDGE) + Reducer 10 <- Map 9 (SIMPLE_EDGE) + Reducer 11 <- Reducer 10 (SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE) + Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Reducer 7 <- Map 6 (SIMPLE_EDGE) Reducer 8 <- Reducer 7 (SIMPLE_EDGE) - Reducer 9 <- Reducer 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1302,11 +1301,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: key, value Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -1316,7 +1315,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 10 + Map 6 Map Operator Tree: TableScan alias: src @@ -1326,11 +1325,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: key, value Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -1340,7 +1339,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Map 6 + Map 9 Map Operator Tree: TableScan alias: src @@ -1350,11 +1349,11 @@ STAGE PLANS: Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + outputColumnNames: key, value Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(_col1) - keys: _col0 (type: string) + aggregations: count(value) + keys: key (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -1364,7 +1363,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 11 + Reducer 10 Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -1384,7 +1383,7 @@ STAGE PLANS: sort order: + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Reducer 12 + Reducer 11 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint) @@ -1430,29 +1429,31 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 condition expressions: 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col2} - outputColumnNames: _col0, _col1, _col2, _col3, _col5 - Statistics: Num rows: 150 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + 1 {KEY.reducesinkkey0} {VALUE._col0} + 2 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: string), _col5 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col0 (type: string), _col1 (type: bigint) + expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 150 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + - Statistics: Num rows: 150 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint) Reducer 5 Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: bigint), VALUE._col3 (type: string), VALUE._col4 (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 150 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 150 Data size: 1606 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1486,26 +1487,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Reducer 9 - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {VALUE._col0} - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col3 (type: bigint) - outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint), _col3 (type: bigint) Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out index 4682a05..aec18a8 100644 --- ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out +++ ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out @@ -113,6 +113,8 @@ STAGE PLANS: 0 key (type: int) 1 key (type: int) outputColumnNames: _col0, _col1 + input vertices: + 1 Map 1 Statistics: Num rows: 550 Data size: 28771 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string) diff --git ql/src/test/results/clientpositive/tez/tez_union.q.out ql/src/test/results/clientpositive/tez/tez_union.q.out index 3ae5d80..0420ce0 100644 --- ql/src/test/results/clientpositive/tez/tez_union.q.out +++ ql/src/test/results/clientpositive/tez/tez_union.q.out @@ -483,29 +483,16 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 6 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 8 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Union 4 (CONTAINS) - Reducer 5 <- Union 4 (SIMPLE_EDGE) + Map 2 <- Map 1 (BROADCAST_EDGE), Map 5 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 7 <- Map 1 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 9 <- Map 1 (BROADCAST_EDGE), Map 10 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s7 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 10 - Map Operator Tree: - TableScan - alias: s8 + alias: s1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -525,10 +512,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 2 + Map 10 Map Operator Tree: TableScan - alias: s1 + alias: s5 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -538,20 +525,10 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 3 + Map 2 Map Operator Tree: TableScan - alias: s6 + alias: s2 Filter Operator predicate: key is not null (type: boolean) Map Join Operator @@ -565,48 +542,63 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 1 + 1 Map 5 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col6 - input vertices: - 1 Map 10 - Select Operator - expressions: _col1 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col5, _col6 + input vertices: + 0 Map 1 + 2 Map 8 + Select Operator + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Map 6 Map Operator Tree: TableScan - alias: s4 + alias: s7 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 7 + Map Operator Tree: + TableScan + alias: s6 Filter Operator predicate: key is not null (type: boolean) Map Join Operator @@ -620,51 +612,37 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 9 + 1 Map 6 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col5, _col6 + input vertices: + 0 Map 1 + 2 Map 8 Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col6 - input vertices: - 1 Map 10 - Select Operator - expressions: _col1 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) - Map 7 + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) + Map 8 Map Operator Tree: TableScan - alias: s3 + alias: s8 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) @@ -674,10 +652,20 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 8 + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 9 Map Operator Tree: TableScan - alias: s2 + alias: s4 Filter Operator predicate: key is not null (type: boolean) Map Join Operator @@ -691,75 +679,48 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 7 + 1 Map 10 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 1 to 2 + condition expressions: + 0 {key} + 1 {_col0} + 2 {key} + keys: + 0 key (type: string) + 1 _col0 (type: string) + 2 key (type: string) + outputColumnNames: _col0, _col5, _col6 + input vertices: + 0 Map 1 + 2 Map 8 Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1 - input vertices: - 1 Map 2 - Map Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {_col0} {_col1} - 1 {key} - keys: - 0 _col0 (type: string) - 1 key (type: string) - outputColumnNames: _col0, _col1, _col6 - input vertices: - 1 Map 10 - Select Operator - expressions: _col1 (type: string), _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1, _col2 - Reduce Output Operator - key expressions: _col2 (type: string) - sort order: + - value expressions: _col0 (type: string), _col1 (type: string) - Map 9 - Map Operator Tree: - TableScan - alias: s5 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reducer 5 + expressions: _col0 (type: string), _col5 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 4 Reduce Operator Tree: Select Operator expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 997 Data size: 10602 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1815 Data size: 19278 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 997 Data size: 10602 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1815 Data size: 19278 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + Union 3 + Vertex: Union 3 Stage: Stage-0 Fetch Operator @@ -923,39 +884,13 @@ STAGE PLANS: Stage: Stage-1 Tez Edges: - Map 3 <- Map 1 (BROADCAST_EDGE), Union 4 (CONTAINS) - Map 5 <- Map 2 (BROADCAST_EDGE), Union 4 (CONTAINS) + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 2 (CONTAINS) #### A masked pattern was here #### Vertices: Map 1 Map Operator Tree: TableScan - alias: s3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 2 - Map Operator Tree: - TableScan - alias: s5 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 3 - Map Operator Tree: - TableScan alias: s2 Filter Operator predicate: key is not null (type: boolean) @@ -970,7 +905,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 1 + 1 Map 3 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -983,7 +918,20 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Map 5 + Map 3 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 4 Map Operator Tree: TableScan alias: s4 @@ -1000,7 +948,7 @@ STAGE PLANS: 1 key (type: string) outputColumnNames: _col0 input vertices: - 1 Map 2 + 1 Map 5 Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -1013,8 +961,21 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Union 4 - Vertex: Union 4 + Map 5 + Map Operator Tree: + TableScan + alias: s5 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out index 6dfcd83..71367c4 100644 --- ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out +++ ql/src/test/results/clientpositive/tez/vector_left_outer_join.q.out @@ -54,6 +54,8 @@ STAGE PLANS: 0 cint (type: int) 1 cint (type: int) outputColumnNames: _col0 + input vertices: + 1 Map 4 Statistics: Num rows: 13516 Data size: 414960 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -64,6 +66,8 @@ STAGE PLANS: keys: 0 _col0 (type: tinyint) 1 ctinyint (type: tinyint) + input vertices: + 1 Map 1 Statistics: Num rows: 14867 Data size: 456456 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 14867 Data size: 456456 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out index 27423b2..4b35231 100644 --- ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out +++ ql/src/test/results/clientpositive/tez/vector_mapjoin_reduce.q.out @@ -204,6 +204,8 @@ STAGE PLANS: 0 _col0 (type: int) 1 l_partkey (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 1 Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -215,6 +217,8 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col3 + input vertices: + 1 Map 4 Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) @@ -363,6 +367,8 @@ STAGE PLANS: 0 _col0 (type: int) 1 l_partkey (type: int) outputColumnNames: _col0, _col1, _col3 + input vertices: + 1 Map 1 Statistics: Num rows: 831 Data size: 3326 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: @@ -374,6 +380,8 @@ STAGE PLANS: 0 _col1 (type: int), 1 (type: int) 1 _col0 (type: int), _col1 (type: int) outputColumnNames: _col0, _col3 + input vertices: + 1 Map 4 Statistics: Num rows: 914 Data size: 3658 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col3 (type: int) diff --git ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out index 7e45d66..5aa9599 100644 --- ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out @@ -50,6 +50,8 @@ STAGE PLANS: 0 cint (type: int) 1 cint (type: int) outputColumnNames: _col2, _col17 + input vertices: + 1 Map 1 Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: int), _col17 (type: int) diff --git ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out index e46c1a9..83c4bb8 100644 --- ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_nested_mapjoin.q.out @@ -33,6 +33,8 @@ STAGE PLANS: 0 _col0 (type: smallint) 1 csmallint (type: smallint) outputColumnNames: _col1 + input vertices: + 0 Map 4 Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double) @@ -81,6 +83,8 @@ STAGE PLANS: 0 ctinyint (type: tinyint) 1 ctinyint (type: tinyint) outputColumnNames: _col0, _col1, _col5, _col15 + input vertices: + 0 Map 3 Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_col0 = _col15) (type: boolean) diff --git ql/src/test/results/clientpositive/udf_if.q.out ql/src/test/results/clientpositive/udf_if.q.out index d696ad7..a2d2c08 100644 --- ql/src/test/results/clientpositive/udf_if.q.out +++ ql/src/test/results/clientpositive/udf_if.q.out @@ -2,12 +2,12 @@ PREHOOK: query: DESCRIBE FUNCTION if PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION if POSTHOOK: type: DESCFUNCTION -There is no documentation for function 'if' +IF(expr1,expr2,expr3) - If expr1 is TRUE (expr1 <> 0 and expr1 <> NULL) then IF() returns expr2; otherwise it returns expr3. IF() returns a numeric or string value, depending on the context in which it is used. PREHOOK: query: DESCRIBE FUNCTION EXTENDED if PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED if POSTHOOK: type: DESCFUNCTION -There is no documentation for function 'if' +IF(expr1,expr2,expr3) - If expr1 is TRUE (expr1 <> 0 and expr1 <> NULL) then IF() returns expr2; otherwise it returns expr3. IF() returns a numeric or string value, depending on the context in which it is used. PREHOOK: query: EXPLAIN SELECT IF(TRUE, 1, 2) AS COL1, IF(FALSE, CAST(NULL AS STRING), CAST(1 AS STRING)) AS COL2,