Index: conf/hive-default.xml.template
===================================================================
--- conf/hive-default.xml.template (revision 1459200)
+++ conf/hive-default.xml.template (working copy)
@@ -785,6 +785,12 @@
+ hive.ignore.mapjoin.hint
+ true
+ Ignore the mapjoin hint
+
+
+
hive.mapjoin.localtask.max.memory.usage
0.90
This number means how much memory the local task can take to hold the key/value into in-memory hash table; If the local task's memory usage is more than this number, the local task will be abort by themself. It means the data of small table is too large to be hold in the memory.
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1459200)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -223,6 +223,9 @@
// not specified
DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", true),
+ // ignore the mapjoin hint
+ HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true),
+
// Hadoop Configuration Properties
// Properties with null values are ignored and exist only for the purpose of giving us
// a symbolic name to reference in the Hive source code. Properties with non-null
Index: ql/src/test/results/clientpositive/join29.q.out
===================================================================
--- ql/src/test/results/clientpositive/join29.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/join29.q.out (working copy)
@@ -375,3 +375,384 @@
406 1 4
66 1 1
98 1 2
+PREHOOK: query: -- Mapjoin followed by group by is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Mapjoin followed by group by is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ]
+POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src1) x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x) key)))) subq1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src) y)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) key)) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL y) key)))) subq2) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq1) cnt)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq2) cnt)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-8, Stage-9, Stage-2
+ Stage-8 has a backup stage: Stage-2
+ Stage-5 depends on stages: Stage-8
+ Stage-0 depends on stages: Stage-2, Stage-5, Stage-6
+ Stage-3 depends on stages: Stage-0
+ Stage-9 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-9
+ Stage-2
+ Stage-4 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq2:y
+ TableScan
+ alias: y
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ expr: UDFToInteger(_col3)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ expr: UDFToInteger(_col3)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: bigint
+ $INTNAME1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ expr: UDFToInteger(_col3)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:x
+ TableScan
+ alias: x
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ]
+POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ]
+POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ]
+POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ]
+POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ]
+POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ]
+POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+128 1 3
+146 1 2
+150 1 1
+213 1 2
+224 1 2
+238 1 2
+255 1 2
+273 1 3
+278 1 2
+311 1 3
+369 1 3
+401 1 5
+406 1 4
+66 1 1
+98 1 2
Index: ql/src/test/results/clientpositive/auto_join29.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join29.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/auto_join29.q.out (working copy)
@@ -20,7 +20,7 @@
src1
Fetch Operator
limit: -1
- src2
+ src3
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
@@ -41,7 +41,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
+ Position of Big Table: 1
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10.0)}
+ 1
+ 2 {(key < 10.0)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
src2
TableScan
alias: src2
@@ -49,7 +71,10 @@
predicate:
expr: (key > 10.0)
type: boolean
- HashTable Sink Operator
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
condition expressions:
0 {key} {value}
1 {key} {value}
@@ -63,54 +88,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
-
- Stage: Stage-5
- Map Reduce
- Alias -> Map Operator Tree:
- src3
- TableScan
- alias: src3
- Map Join Operator
- condition map:
- Left Outer Join0 to 1
- Right Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
- filter predicates:
- 0 {(key < 10.0)}
- 1
- 2 {(key < 10.0)}
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- 2 [Column[key]]
- outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
- Position of Big Table: 2
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- expr: _col4
- type: string
- expr: _col5
- type: string
- expr: _col8
- type: string
- expr: _col9
- type: string
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Local Work:
Map Reduce Local Work
@@ -169,10 +169,6 @@
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 100 val_100
NULL NULL NULL NULL 100 val_100
NULL NULL NULL NULL 100 val_100
@@ -707,7 +703,6 @@
NULL NULL NULL NULL 199 val_199
NULL NULL NULL NULL 199 val_199
NULL NULL NULL NULL 199 val_199
-NULL NULL NULL NULL 2 val_2
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 200 val_200
NULL NULL NULL NULL 200 val_200
@@ -1757,7 +1752,6 @@
NULL NULL NULL NULL 399 val_399
NULL NULL NULL NULL 399 val_399
NULL NULL NULL NULL 399 val_399
-NULL NULL NULL NULL 4 val_4
NULL NULL NULL NULL 400 val_400
NULL NULL NULL NULL 401 val_401
NULL NULL NULL NULL 401 val_401
@@ -2619,9 +2613,6 @@
NULL NULL NULL NULL 498 val_498
NULL NULL NULL NULL 498 val_498
NULL NULL NULL NULL 498 val_498
-NULL NULL NULL NULL 5 val_5
-NULL NULL NULL NULL 5 val_5
-NULL NULL NULL NULL 5 val_5
NULL NULL NULL NULL 51 val_51
NULL NULL NULL NULL 51 val_51
NULL NULL NULL NULL 51 val_51
@@ -2699,7 +2690,6 @@
NULL NULL NULL NULL 76 val_76
NULL NULL NULL NULL 77 val_77
NULL NULL NULL NULL 78 val_78
-NULL NULL NULL NULL 8 val_8
NULL NULL NULL NULL 80 val_80
NULL NULL NULL NULL 82 val_82
NULL NULL NULL NULL 83 val_83
@@ -2721,7 +2711,6 @@
NULL NULL NULL NULL 85 val_85
NULL NULL NULL NULL 86 val_86
NULL NULL NULL NULL 87 val_87
-NULL NULL NULL NULL 9 val_9
NULL NULL NULL NULL 90 val_90
NULL NULL NULL NULL 90 val_90
NULL NULL NULL NULL 90 val_90
@@ -2794,19 +2783,37 @@
Stage: Stage-6
Map Reduce Local Work
Alias -> Map Local Tables:
- src2
+ src1
Fetch Operator
limit: -1
src3
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- src2
+ src1
TableScan
- alias: src2
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10.0)}
+ 1
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+ src3
+ TableScan
+ alias: src3
Filter Operator
predicate:
- expr: (key > 10.0)
+ expr: ((key > 10.0) and (key < 10.0))
type: boolean
HashTable Sink Operator
condition expressions:
@@ -2822,15 +2829,22 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 0
- src3
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
TableScan
- alias: src3
+ alias: src2
Filter Operator
predicate:
- expr: ((key > 10.0) and (key < 10.0))
+ expr: (key > 10.0)
type: boolean
- HashTable Sink Operator
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join1 to 2
condition expressions:
0 {key} {value}
1 {key} {value}
@@ -2844,54 +2858,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 0
-
- Stage: Stage-5
- Map Reduce
- Alias -> Map Operator Tree:
- src1
- TableScan
- alias: src1
- Map Join Operator
- condition map:
- Left Outer Join0 to 1
- Left Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
- filter predicates:
- 0 {(key < 10.0)}
- 1
- 2
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- 2 [Column[key]]
- outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
- Position of Big Table: 0
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- expr: _col4
- type: string
- expr: _col5
- type: string
- expr: _col8
- type: string
- expr: _col9
- type: string
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Local Work:
Map Reduce Local Work
@@ -2950,10 +2939,6 @@
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0 NULL NULL NULL NULL
-0 val_0 NULL NULL NULL NULL
-0 val_0 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
100 val_100 NULL NULL NULL NULL
100 val_100 NULL NULL NULL NULL
100 val_100 NULL NULL NULL NULL
@@ -3180,7 +3165,6 @@
199 val_199 NULL NULL NULL NULL
199 val_199 NULL NULL NULL NULL
199 val_199 NULL NULL NULL NULL
-2 val_2 NULL NULL NULL NULL
20 val_20 NULL NULL NULL NULL
200 val_200 NULL NULL NULL NULL
200 val_200 NULL NULL NULL NULL
@@ -3600,7 +3584,6 @@
399 val_399 NULL NULL NULL NULL
399 val_399 NULL NULL NULL NULL
399 val_399 NULL NULL NULL NULL
-4 val_4 NULL NULL NULL NULL
400 val_400 NULL NULL NULL NULL
401 val_401 NULL NULL NULL NULL
401 val_401 NULL NULL NULL NULL
@@ -3886,9 +3869,6 @@
498 val_498 NULL NULL NULL NULL
498 val_498 NULL NULL NULL NULL
498 val_498 NULL NULL NULL NULL
-5 val_5 NULL NULL NULL NULL
-5 val_5 NULL NULL NULL NULL
-5 val_5 NULL NULL NULL NULL
51 val_51 NULL NULL NULL NULL
51 val_51 NULL NULL NULL NULL
51 val_51 NULL NULL NULL NULL
@@ -3928,7 +3908,6 @@
76 val_76 NULL NULL NULL NULL
77 val_77 NULL NULL NULL NULL
78 val_78 NULL NULL NULL NULL
-8 val_8 NULL NULL NULL NULL
80 val_80 NULL NULL NULL NULL
82 val_82 NULL NULL NULL NULL
83 val_83 NULL NULL NULL NULL
@@ -3942,7 +3921,6 @@
85 val_85 NULL NULL NULL NULL
86 val_86 NULL NULL NULL NULL
87 val_87 NULL NULL NULL NULL
-9 val_9 NULL NULL NULL NULL
90 val_90 NULL NULL NULL NULL
90 val_90 NULL NULL NULL NULL
90 val_90 NULL NULL NULL NULL
@@ -4711,7 +4689,7 @@
src1
Fetch Operator
limit: -1
- src2
+ src3
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
@@ -4736,10 +4714,10 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
- src2
+ Position of Big Table: 1
+ src3
TableScan
- alias: src2
+ alias: src3
HashTable Sink Operator
condition expressions:
0 {key} {value}
@@ -4754,14 +4732,14 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
+ Position of Big Table: 1
Stage: Stage-5
Map Reduce
Alias -> Map Operator Tree:
- src3
+ src2
TableScan
- alias: src3
+ alias: src2
Map Join Operator
condition map:
Right Outer Join0 to 1
@@ -4780,7 +4758,7 @@
1 [Column[key]]
2 [Column[key]]
outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
- Position of Big Table: 2
+ Position of Big Table: 1
Select Operator
expressions:
expr: _col0
@@ -6113,7 +6091,7 @@
src1
Fetch Operator
limit: -1
- src2
+ src3
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
@@ -6138,7 +6116,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
+ Position of Big Table: 1
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 10.0)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
src2
TableScan
alias: src2
@@ -6146,7 +6146,10 @@
predicate:
expr: ((key < 10.0) and (key > 10.0))
type: boolean
- HashTable Sink Operator
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join1 to 2
condition expressions:
0 {key} {value}
1 {key} {value}
@@ -6160,54 +6163,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 2
-
- Stage: Stage-5
- Map Reduce
- Alias -> Map Operator Tree:
- src3
- TableScan
- alias: src3
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- Right Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
- filter predicates:
- 0
- 1
- 2 {(key < 10.0)}
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- 2 [Column[key]]
- outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
- Position of Big Table: 2
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- expr: _col4
- type: string
- expr: _col5
- type: string
- expr: _col8
- type: string
- expr: _col9
- type: string
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Local Work:
Map Reduce Local Work
@@ -6266,506 +6244,6 @@
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 0 val_0
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 100 val_100
-NULL NULL NULL NULL 100 val_100
-NULL NULL NULL NULL 103 val_103
-NULL NULL NULL NULL 103 val_103
-NULL NULL NULL NULL 104 val_104
-NULL NULL NULL NULL 104 val_104
-NULL NULL NULL NULL 105 val_105
-NULL NULL NULL NULL 11 val_11
-NULL NULL NULL NULL 111 val_111
-NULL NULL NULL NULL 113 val_113
-NULL NULL NULL NULL 113 val_113
-NULL NULL NULL NULL 114 val_114
-NULL NULL NULL NULL 116 val_116
-NULL NULL NULL NULL 118 val_118
-NULL NULL NULL NULL 118 val_118
-NULL NULL NULL NULL 119 val_119
-NULL NULL NULL NULL 119 val_119
-NULL NULL NULL NULL 119 val_119
-NULL NULL NULL NULL 12 val_12
-NULL NULL NULL NULL 12 val_12
-NULL NULL NULL NULL 120 val_120
-NULL NULL NULL NULL 120 val_120
-NULL NULL NULL NULL 125 val_125
-NULL NULL NULL NULL 125 val_125
-NULL NULL NULL NULL 126 val_126
-NULL NULL NULL NULL 128 val_128
-NULL NULL NULL NULL 128 val_128
-NULL NULL NULL NULL 128 val_128
-NULL NULL NULL NULL 129 val_129
-NULL NULL NULL NULL 129 val_129
-NULL NULL NULL NULL 131 val_131
-NULL NULL NULL NULL 133 val_133
-NULL NULL NULL NULL 134 val_134
-NULL NULL NULL NULL 134 val_134
-NULL NULL NULL NULL 136 val_136
-NULL NULL NULL NULL 137 val_137
-NULL NULL NULL NULL 137 val_137
-NULL NULL NULL NULL 138 val_138
-NULL NULL NULL NULL 138 val_138
-NULL NULL NULL NULL 138 val_138
-NULL NULL NULL NULL 138 val_138
-NULL NULL NULL NULL 143 val_143
-NULL NULL NULL NULL 145 val_145
-NULL NULL NULL NULL 146 val_146
-NULL NULL NULL NULL 146 val_146
-NULL NULL NULL NULL 149 val_149
-NULL NULL NULL NULL 149 val_149
-NULL NULL NULL NULL 15 val_15
-NULL NULL NULL NULL 15 val_15
-NULL NULL NULL NULL 150 val_150
-NULL NULL NULL NULL 152 val_152
-NULL NULL NULL NULL 152 val_152
-NULL NULL NULL NULL 153 val_153
-NULL NULL NULL NULL 155 val_155
-NULL NULL NULL NULL 156 val_156
-NULL NULL NULL NULL 157 val_157
-NULL NULL NULL NULL 158 val_158
-NULL NULL NULL NULL 160 val_160
-NULL NULL NULL NULL 162 val_162
-NULL NULL NULL NULL 163 val_163
-NULL NULL NULL NULL 164 val_164
-NULL NULL NULL NULL 164 val_164
-NULL NULL NULL NULL 165 val_165
-NULL NULL NULL NULL 165 val_165
-NULL NULL NULL NULL 166 val_166
-NULL NULL NULL NULL 167 val_167
-NULL NULL NULL NULL 167 val_167
-NULL NULL NULL NULL 167 val_167
-NULL NULL NULL NULL 168 val_168
-NULL NULL NULL NULL 169 val_169
-NULL NULL NULL NULL 169 val_169
-NULL NULL NULL NULL 169 val_169
-NULL NULL NULL NULL 169 val_169
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 170 val_170
-NULL NULL NULL NULL 172 val_172
-NULL NULL NULL NULL 172 val_172
-NULL NULL NULL NULL 174 val_174
-NULL NULL NULL NULL 174 val_174
-NULL NULL NULL NULL 175 val_175
-NULL NULL NULL NULL 175 val_175
-NULL NULL NULL NULL 176 val_176
-NULL NULL NULL NULL 176 val_176
-NULL NULL NULL NULL 177 val_177
-NULL NULL NULL NULL 178 val_178
-NULL NULL NULL NULL 179 val_179
-NULL NULL NULL NULL 179 val_179
-NULL NULL NULL NULL 18 val_18
-NULL NULL NULL NULL 18 val_18
-NULL NULL NULL NULL 180 val_180
-NULL NULL NULL NULL 181 val_181
-NULL NULL NULL NULL 183 val_183
-NULL NULL NULL NULL 186 val_186
-NULL NULL NULL NULL 187 val_187
-NULL NULL NULL NULL 187 val_187
-NULL NULL NULL NULL 187 val_187
-NULL NULL NULL NULL 189 val_189
-NULL NULL NULL NULL 19 val_19
-NULL NULL NULL NULL 190 val_190
-NULL NULL NULL NULL 191 val_191
-NULL NULL NULL NULL 191 val_191
-NULL NULL NULL NULL 192 val_192
-NULL NULL NULL NULL 193 val_193
-NULL NULL NULL NULL 193 val_193
-NULL NULL NULL NULL 193 val_193
-NULL NULL NULL NULL 194 val_194
-NULL NULL NULL NULL 195 val_195
-NULL NULL NULL NULL 195 val_195
-NULL NULL NULL NULL 196 val_196
-NULL NULL NULL NULL 197 val_197
-NULL NULL NULL NULL 197 val_197
-NULL NULL NULL NULL 199 val_199
-NULL NULL NULL NULL 199 val_199
-NULL NULL NULL NULL 199 val_199
-NULL NULL NULL NULL 2 val_2
-NULL NULL NULL NULL 20 val_20
-NULL NULL NULL NULL 200 val_200
-NULL NULL NULL NULL 200 val_200
-NULL NULL NULL NULL 201 val_201
-NULL NULL NULL NULL 202 val_202
-NULL NULL NULL NULL 203 val_203
-NULL NULL NULL NULL 203 val_203
-NULL NULL NULL NULL 205 val_205
-NULL NULL NULL NULL 205 val_205
-NULL NULL NULL NULL 207 val_207
-NULL NULL NULL NULL 207 val_207
-NULL NULL NULL NULL 208 val_208
-NULL NULL NULL NULL 208 val_208
-NULL NULL NULL NULL 208 val_208
-NULL NULL NULL NULL 209 val_209
-NULL NULL NULL NULL 209 val_209
-NULL NULL NULL NULL 213 val_213
-NULL NULL NULL NULL 213 val_213
-NULL NULL NULL NULL 214 val_214
-NULL NULL NULL NULL 216 val_216
-NULL NULL NULL NULL 216 val_216
-NULL NULL NULL NULL 217 val_217
-NULL NULL NULL NULL 217 val_217
-NULL NULL NULL NULL 218 val_218
-NULL NULL NULL NULL 219 val_219
-NULL NULL NULL NULL 219 val_219
-NULL NULL NULL NULL 221 val_221
-NULL NULL NULL NULL 221 val_221
-NULL NULL NULL NULL 222 val_222
-NULL NULL NULL NULL 223 val_223
-NULL NULL NULL NULL 223 val_223
-NULL NULL NULL NULL 224 val_224
-NULL NULL NULL NULL 224 val_224
-NULL NULL NULL NULL 226 val_226
-NULL NULL NULL NULL 228 val_228
-NULL NULL NULL NULL 229 val_229
-NULL NULL NULL NULL 229 val_229
-NULL NULL NULL NULL 230 val_230
-NULL NULL NULL NULL 230 val_230
-NULL NULL NULL NULL 230 val_230
-NULL NULL NULL NULL 230 val_230
-NULL NULL NULL NULL 230 val_230
-NULL NULL NULL NULL 233 val_233
-NULL NULL NULL NULL 233 val_233
-NULL NULL NULL NULL 235 val_235
-NULL NULL NULL NULL 237 val_237
-NULL NULL NULL NULL 237 val_237
-NULL NULL NULL NULL 238 val_238
-NULL NULL NULL NULL 238 val_238
-NULL NULL NULL NULL 239 val_239
-NULL NULL NULL NULL 239 val_239
-NULL NULL NULL NULL 24 val_24
-NULL NULL NULL NULL 24 val_24
-NULL NULL NULL NULL 241 val_241
-NULL NULL NULL NULL 242 val_242
-NULL NULL NULL NULL 242 val_242
-NULL NULL NULL NULL 244 val_244
-NULL NULL NULL NULL 247 val_247
-NULL NULL NULL NULL 248 val_248
-NULL NULL NULL NULL 249 val_249
-NULL NULL NULL NULL 252 val_252
-NULL NULL NULL NULL 255 val_255
-NULL NULL NULL NULL 255 val_255
-NULL NULL NULL NULL 256 val_256
-NULL NULL NULL NULL 256 val_256
-NULL NULL NULL NULL 257 val_257
-NULL NULL NULL NULL 258 val_258
-NULL NULL NULL NULL 26 val_26
-NULL NULL NULL NULL 26 val_26
-NULL NULL NULL NULL 260 val_260
-NULL NULL NULL NULL 262 val_262
-NULL NULL NULL NULL 263 val_263
-NULL NULL NULL NULL 265 val_265
-NULL NULL NULL NULL 265 val_265
-NULL NULL NULL NULL 266 val_266
-NULL NULL NULL NULL 27 val_27
-NULL NULL NULL NULL 272 val_272
-NULL NULL NULL NULL 272 val_272
-NULL NULL NULL NULL 273 val_273
-NULL NULL NULL NULL 273 val_273
-NULL NULL NULL NULL 273 val_273
-NULL NULL NULL NULL 274 val_274
-NULL NULL NULL NULL 275 val_275
-NULL NULL NULL NULL 277 val_277
-NULL NULL NULL NULL 277 val_277
-NULL NULL NULL NULL 277 val_277
-NULL NULL NULL NULL 277 val_277
-NULL NULL NULL NULL 278 val_278
-NULL NULL NULL NULL 278 val_278
-NULL NULL NULL NULL 28 val_28
-NULL NULL NULL NULL 280 val_280
-NULL NULL NULL NULL 280 val_280
-NULL NULL NULL NULL 281 val_281
-NULL NULL NULL NULL 281 val_281
-NULL NULL NULL NULL 282 val_282
-NULL NULL NULL NULL 282 val_282
-NULL NULL NULL NULL 283 val_283
-NULL NULL NULL NULL 284 val_284
-NULL NULL NULL NULL 285 val_285
-NULL NULL NULL NULL 286 val_286
-NULL NULL NULL NULL 287 val_287
-NULL NULL NULL NULL 288 val_288
-NULL NULL NULL NULL 288 val_288
-NULL NULL NULL NULL 289 val_289
-NULL NULL NULL NULL 291 val_291
-NULL NULL NULL NULL 292 val_292
-NULL NULL NULL NULL 296 val_296
-NULL NULL NULL NULL 298 val_298
-NULL NULL NULL NULL 298 val_298
-NULL NULL NULL NULL 298 val_298
-NULL NULL NULL NULL 30 val_30
-NULL NULL NULL NULL 302 val_302
-NULL NULL NULL NULL 305 val_305
-NULL NULL NULL NULL 306 val_306
-NULL NULL NULL NULL 307 val_307
-NULL NULL NULL NULL 307 val_307
-NULL NULL NULL NULL 308 val_308
-NULL NULL NULL NULL 309 val_309
-NULL NULL NULL NULL 309 val_309
-NULL NULL NULL NULL 310 val_310
-NULL NULL NULL NULL 311 val_311
-NULL NULL NULL NULL 311 val_311
-NULL NULL NULL NULL 311 val_311
-NULL NULL NULL NULL 315 val_315
-NULL NULL NULL NULL 316 val_316
-NULL NULL NULL NULL 316 val_316
-NULL NULL NULL NULL 316 val_316
-NULL NULL NULL NULL 317 val_317
-NULL NULL NULL NULL 317 val_317
-NULL NULL NULL NULL 318 val_318
-NULL NULL NULL NULL 318 val_318
-NULL NULL NULL NULL 318 val_318
-NULL NULL NULL NULL 321 val_321
-NULL NULL NULL NULL 321 val_321
-NULL NULL NULL NULL 322 val_322
-NULL NULL NULL NULL 322 val_322
-NULL NULL NULL NULL 323 val_323
-NULL NULL NULL NULL 325 val_325
-NULL NULL NULL NULL 325 val_325
-NULL NULL NULL NULL 327 val_327
-NULL NULL NULL NULL 327 val_327
-NULL NULL NULL NULL 327 val_327
-NULL NULL NULL NULL 33 val_33
-NULL NULL NULL NULL 331 val_331
-NULL NULL NULL NULL 331 val_331
-NULL NULL NULL NULL 332 val_332
-NULL NULL NULL NULL 333 val_333
-NULL NULL NULL NULL 333 val_333
-NULL NULL NULL NULL 335 val_335
-NULL NULL NULL NULL 336 val_336
-NULL NULL NULL NULL 338 val_338
-NULL NULL NULL NULL 339 val_339
-NULL NULL NULL NULL 34 val_34
-NULL NULL NULL NULL 341 val_341
-NULL NULL NULL NULL 342 val_342
-NULL NULL NULL NULL 342 val_342
-NULL NULL NULL NULL 344 val_344
-NULL NULL NULL NULL 344 val_344
-NULL NULL NULL NULL 345 val_345
-NULL NULL NULL NULL 348 val_348
-NULL NULL NULL NULL 348 val_348
-NULL NULL NULL NULL 348 val_348
-NULL NULL NULL NULL 348 val_348
-NULL NULL NULL NULL 348 val_348
-NULL NULL NULL NULL 35 val_35
-NULL NULL NULL NULL 35 val_35
-NULL NULL NULL NULL 35 val_35
-NULL NULL NULL NULL 351 val_351
-NULL NULL NULL NULL 353 val_353
-NULL NULL NULL NULL 353 val_353
-NULL NULL NULL NULL 356 val_356
-NULL NULL NULL NULL 360 val_360
-NULL NULL NULL NULL 362 val_362
-NULL NULL NULL NULL 364 val_364
-NULL NULL NULL NULL 365 val_365
-NULL NULL NULL NULL 366 val_366
-NULL NULL NULL NULL 367 val_367
-NULL NULL NULL NULL 367 val_367
-NULL NULL NULL NULL 368 val_368
-NULL NULL NULL NULL 369 val_369
-NULL NULL NULL NULL 369 val_369
-NULL NULL NULL NULL 369 val_369
-NULL NULL NULL NULL 37 val_37
-NULL NULL NULL NULL 37 val_37
-NULL NULL NULL NULL 373 val_373
-NULL NULL NULL NULL 374 val_374
-NULL NULL NULL NULL 375 val_375
-NULL NULL NULL NULL 377 val_377
-NULL NULL NULL NULL 378 val_378
-NULL NULL NULL NULL 379 val_379
-NULL NULL NULL NULL 382 val_382
-NULL NULL NULL NULL 382 val_382
-NULL NULL NULL NULL 384 val_384
-NULL NULL NULL NULL 384 val_384
-NULL NULL NULL NULL 384 val_384
-NULL NULL NULL NULL 386 val_386
-NULL NULL NULL NULL 389 val_389
-NULL NULL NULL NULL 392 val_392
-NULL NULL NULL NULL 393 val_393
-NULL NULL NULL NULL 394 val_394
-NULL NULL NULL NULL 395 val_395
-NULL NULL NULL NULL 395 val_395
-NULL NULL NULL NULL 396 val_396
-NULL NULL NULL NULL 396 val_396
-NULL NULL NULL NULL 396 val_396
-NULL NULL NULL NULL 397 val_397
-NULL NULL NULL NULL 397 val_397
-NULL NULL NULL NULL 399 val_399
-NULL NULL NULL NULL 399 val_399
-NULL NULL NULL NULL 4 val_4
-NULL NULL NULL NULL 400 val_400
-NULL NULL NULL NULL 401 val_401
-NULL NULL NULL NULL 401 val_401
-NULL NULL NULL NULL 401 val_401
-NULL NULL NULL NULL 401 val_401
-NULL NULL NULL NULL 401 val_401
-NULL NULL NULL NULL 402 val_402
-NULL NULL NULL NULL 403 val_403
-NULL NULL NULL NULL 403 val_403
-NULL NULL NULL NULL 403 val_403
-NULL NULL NULL NULL 404 val_404
-NULL NULL NULL NULL 404 val_404
-NULL NULL NULL NULL 406 val_406
-NULL NULL NULL NULL 406 val_406
-NULL NULL NULL NULL 406 val_406
-NULL NULL NULL NULL 406 val_406
-NULL NULL NULL NULL 407 val_407
-NULL NULL NULL NULL 409 val_409
-NULL NULL NULL NULL 409 val_409
-NULL NULL NULL NULL 409 val_409
-NULL NULL NULL NULL 41 val_41
-NULL NULL NULL NULL 411 val_411
-NULL NULL NULL NULL 413 val_413
-NULL NULL NULL NULL 413 val_413
-NULL NULL NULL NULL 414 val_414
-NULL NULL NULL NULL 414 val_414
-NULL NULL NULL NULL 417 val_417
-NULL NULL NULL NULL 417 val_417
-NULL NULL NULL NULL 417 val_417
-NULL NULL NULL NULL 418 val_418
-NULL NULL NULL NULL 419 val_419
-NULL NULL NULL NULL 42 val_42
-NULL NULL NULL NULL 42 val_42
-NULL NULL NULL NULL 421 val_421
-NULL NULL NULL NULL 424 val_424
-NULL NULL NULL NULL 424 val_424
-NULL NULL NULL NULL 427 val_427
-NULL NULL NULL NULL 429 val_429
-NULL NULL NULL NULL 429 val_429
-NULL NULL NULL NULL 43 val_43
-NULL NULL NULL NULL 430 val_430
-NULL NULL NULL NULL 430 val_430
-NULL NULL NULL NULL 430 val_430
-NULL NULL NULL NULL 431 val_431
-NULL NULL NULL NULL 431 val_431
-NULL NULL NULL NULL 431 val_431
-NULL NULL NULL NULL 432 val_432
-NULL NULL NULL NULL 435 val_435
-NULL NULL NULL NULL 436 val_436
-NULL NULL NULL NULL 437 val_437
-NULL NULL NULL NULL 438 val_438
-NULL NULL NULL NULL 438 val_438
-NULL NULL NULL NULL 438 val_438
-NULL NULL NULL NULL 439 val_439
-NULL NULL NULL NULL 439 val_439
-NULL NULL NULL NULL 44 val_44
-NULL NULL NULL NULL 443 val_443
-NULL NULL NULL NULL 444 val_444
-NULL NULL NULL NULL 446 val_446
-NULL NULL NULL NULL 448 val_448
-NULL NULL NULL NULL 449 val_449
-NULL NULL NULL NULL 452 val_452
-NULL NULL NULL NULL 453 val_453
-NULL NULL NULL NULL 454 val_454
-NULL NULL NULL NULL 454 val_454
-NULL NULL NULL NULL 454 val_454
-NULL NULL NULL NULL 455 val_455
-NULL NULL NULL NULL 457 val_457
-NULL NULL NULL NULL 458 val_458
-NULL NULL NULL NULL 458 val_458
-NULL NULL NULL NULL 459 val_459
-NULL NULL NULL NULL 459 val_459
-NULL NULL NULL NULL 460 val_460
-NULL NULL NULL NULL 462 val_462
-NULL NULL NULL NULL 462 val_462
-NULL NULL NULL NULL 463 val_463
-NULL NULL NULL NULL 463 val_463
-NULL NULL NULL NULL 466 val_466
-NULL NULL NULL NULL 466 val_466
-NULL NULL NULL NULL 466 val_466
-NULL NULL NULL NULL 467 val_467
-NULL NULL NULL NULL 468 val_468
-NULL NULL NULL NULL 468 val_468
-NULL NULL NULL NULL 468 val_468
-NULL NULL NULL NULL 468 val_468
-NULL NULL NULL NULL 469 val_469
-NULL NULL NULL NULL 469 val_469
-NULL NULL NULL NULL 469 val_469
-NULL NULL NULL NULL 469 val_469
-NULL NULL NULL NULL 469 val_469
-NULL NULL NULL NULL 47 val_47
-NULL NULL NULL NULL 470 val_470
-NULL NULL NULL NULL 472 val_472
-NULL NULL NULL NULL 475 val_475
-NULL NULL NULL NULL 477 val_477
-NULL NULL NULL NULL 478 val_478
-NULL NULL NULL NULL 478 val_478
-NULL NULL NULL NULL 479 val_479
-NULL NULL NULL NULL 480 val_480
-NULL NULL NULL NULL 480 val_480
-NULL NULL NULL NULL 480 val_480
-NULL NULL NULL NULL 481 val_481
-NULL NULL NULL NULL 482 val_482
-NULL NULL NULL NULL 483 val_483
-NULL NULL NULL NULL 484 val_484
-NULL NULL NULL NULL 485 val_485
-NULL NULL NULL NULL 487 val_487
-NULL NULL NULL NULL 489 val_489
-NULL NULL NULL NULL 489 val_489
-NULL NULL NULL NULL 489 val_489
-NULL NULL NULL NULL 489 val_489
-NULL NULL NULL NULL 490 val_490
-NULL NULL NULL NULL 491 val_491
-NULL NULL NULL NULL 492 val_492
-NULL NULL NULL NULL 492 val_492
-NULL NULL NULL NULL 493 val_493
-NULL NULL NULL NULL 494 val_494
-NULL NULL NULL NULL 495 val_495
-NULL NULL NULL NULL 496 val_496
-NULL NULL NULL NULL 497 val_497
-NULL NULL NULL NULL 498 val_498
-NULL NULL NULL NULL 498 val_498
-NULL NULL NULL NULL 498 val_498
-NULL NULL NULL NULL 5 val_5
-NULL NULL NULL NULL 5 val_5
-NULL NULL NULL NULL 5 val_5
-NULL NULL NULL NULL 51 val_51
-NULL NULL NULL NULL 51 val_51
-NULL NULL NULL NULL 53 val_53
-NULL NULL NULL NULL 54 val_54
-NULL NULL NULL NULL 57 val_57
-NULL NULL NULL NULL 58 val_58
-NULL NULL NULL NULL 58 val_58
-NULL NULL NULL NULL 64 val_64
-NULL NULL NULL NULL 65 val_65
-NULL NULL NULL NULL 66 val_66
-NULL NULL NULL NULL 67 val_67
-NULL NULL NULL NULL 67 val_67
-NULL NULL NULL NULL 69 val_69
-NULL NULL NULL NULL 70 val_70
-NULL NULL NULL NULL 70 val_70
-NULL NULL NULL NULL 70 val_70
-NULL NULL NULL NULL 72 val_72
-NULL NULL NULL NULL 72 val_72
-NULL NULL NULL NULL 74 val_74
-NULL NULL NULL NULL 76 val_76
-NULL NULL NULL NULL 76 val_76
-NULL NULL NULL NULL 77 val_77
-NULL NULL NULL NULL 78 val_78
-NULL NULL NULL NULL 8 val_8
-NULL NULL NULL NULL 80 val_80
-NULL NULL NULL NULL 82 val_82
-NULL NULL NULL NULL 83 val_83
-NULL NULL NULL NULL 83 val_83
-NULL NULL NULL NULL 84 val_84
-NULL NULL NULL NULL 84 val_84
-NULL NULL NULL NULL 85 val_85
-NULL NULL NULL NULL 86 val_86
-NULL NULL NULL NULL 87 val_87
-NULL NULL NULL NULL 9 val_9
-NULL NULL NULL NULL 90 val_90
-NULL NULL NULL NULL 90 val_90
-NULL NULL NULL NULL 90 val_90
-NULL NULL NULL NULL 92 val_92
-NULL NULL NULL NULL 95 val_95
-NULL NULL NULL NULL 95 val_95
-NULL NULL NULL NULL 96 val_96
-NULL NULL NULL NULL 97 val_97
-NULL NULL NULL NULL 97 val_97
-NULL NULL NULL NULL 98 val_98
-NULL NULL NULL NULL 98 val_98
PREHOOK: query: explain
SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
PREHOOK: type: QUERY
@@ -6785,19 +6263,37 @@
Stage: Stage-6
Map Reduce Local Work
Alias -> Map Local Tables:
- src2
+ src1
Fetch Operator
limit: -1
src3
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- src2
+ src1
TableScan
- alias: src2
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10.0)}
+ 1
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+ src3
+ TableScan
+ alias: src3
Filter Operator
predicate:
- expr: ((key < 10.0) and (key > 10.0))
+ expr: ((key > 10.0) and (key < 10.0))
type: boolean
HashTable Sink Operator
condition expressions:
@@ -6813,15 +6309,22 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 0
- src3
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
TableScan
- alias: src3
+ alias: src2
Filter Operator
predicate:
- expr: ((key > 10.0) and (key < 10.0))
+ expr: ((key < 10.0) and (key > 10.0))
type: boolean
- HashTable Sink Operator
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Inner Join 1 to 2
condition expressions:
0 {key} {value}
1 {key} {value}
@@ -6835,54 +6338,29 @@
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- Position of Big Table: 0
-
- Stage: Stage-5
- Map Reduce
- Alias -> Map Operator Tree:
- src1
- TableScan
- alias: src1
- Map Join Operator
- condition map:
- Left Outer Join0 to 1
- Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
- filter predicates:
- 0 {(key < 10.0)}
- 1
- 2
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- 2 [Column[key]]
- outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
- Position of Big Table: 0
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- expr: _col4
- type: string
- expr: _col5
- type: string
- expr: _col8
- type: string
- expr: _col9
- type: string
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Local Work:
Map Reduce Local Work
Index: ql/src/test/results/clientpositive/join35.q.out
===================================================================
--- ql/src/test/results/clientpositive/join35.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/join35.q.out (working copy)
@@ -916,3 +916,950 @@
369 3
401 val_401 5
406 val_406 4
+PREHOOK: query: -- Mapjoin followed by union is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Mapjoin followed by union is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src) x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL x) key) 20)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x) key)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src) x1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x1) key) key) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_WHERE (> (. (TOK_TABLE_OR_COL x1) key) 100)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x1) key))))) subq1) (TOK_TABREF (TOK_TABNAME src1) x) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL subq1) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq1) cnt)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-8 depends on stages: Stage-1, Stage-4 , consists of Stage-9, Stage-10, Stage-2
+ Stage-9 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-9
+ Stage-0 depends on stages: Stage-2, Stage-6, Stage-7
+ Stage-3 depends on stages: Stage-0
+ Stage-10 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-10
+ Stage-2
+ Stage-4 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ null-subquery1:subq1-subquery1:x
+ TableScan
+ alias: x
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: (key < 20.0)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /src [null-subquery1:subq1-subquery1:x]
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ x
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ x
+ TableScan
+ alias: x
+ GatherStats: false
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ outputColumnNames: _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:int
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 2
+ numPartitions 0
+ numRows 13
+ rawDataSize 148
+ serialization.ddl struct dest_j1 { string key, string value, i32 val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 161
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ outputColumnNames: _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:int
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 2
+ numPartitions 0
+ numRows 13
+ rawDataSize 148
+ serialization.ddl struct dest_j1 { string key, string value, i32 val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 161
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Local Work:
+ Map Reduce Local Work
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10003
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: src1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src1
+ name: default.src1
+ Truncated Path -> Alias:
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:int
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 2
+ numPartitions 0
+ numRows 13
+ rawDataSize 148
+ serialization.ddl struct dest_j1 { string key, string value, i32 val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 161
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+#### A masked pattern was here ####
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+#### A masked pattern was here ####
+ Fetch Operator
+ limit: -1
+#### A masked pattern was here ####
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ x
+ TableScan
+ alias: x
+ GatherStats: false
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col1}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ outputColumnNames: _col1, _col2, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:int
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 2
+ numPartitions 0
+ numRows 13
+ rawDataSize 148
+ serialization.ddl struct dest_j1 { string key, string value, i32 val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 161
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Local Work:
+ Map Reduce Local Work
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10003
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: src1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src1
+ name: default.src1
+ Truncated Path -> Alias:
+ /src1 [x]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
+ x
+ TableScan
+ alias: x
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10003
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: src1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src1
+ name: default.src1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: UDFToInteger(_col1)
+ type: int
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:int
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 2
+ numPartitions 0
+ numRows 13
+ rawDataSize 148
+ serialization.ddl struct dest_j1 { string key, string value, i32 val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 161
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /src1 [x]
+#### A masked pattern was here ####
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ null-subquery2:subq1-subquery2:x1
+ TableScan
+ alias: x1
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: (key > 100.0)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string,bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /src [null-subquery2:subq1-subquery2:x1]
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+128 3
+146 val_146 2
+150 val_150 1
+213 val_213 2
+224 2
+238 val_238 2
+255 val_255 2
+273 val_273 3
+278 val_278 2
+311 val_311 3
+369 3
+401 val_401 5
+406 val_406 4
Index: ql/src/test/results/clientpositive/bucketcontext_4.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_4.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_4.q.out (working copy)
@@ -263,41 +263,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -349,7 +348,168 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 8
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 8
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_small/ds=2008-04-08 [a]
+ /bucket_small/ds=2008-04-09 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -379,7 +539,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/join28.q.out
===================================================================
--- ql/src/test/results/clientpositive/join28.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/join28.q.out (working copy)
@@ -278,3 +278,284 @@
98 val_98
98 val_98
98 val_98
+PREHOOK: query: -- Mapjoin followed by mapjoin is not supported.
+-- The same query would work fine without the hint, or if the hint is ignored.
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Mapjoin followed by mapjoin is not supported.
+-- The same query would work fine without the hint, or if the hint is ignored.
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value) value1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) key) key2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value) value2)))) subq) (TOK_TABREF (TOK_TABNAME srcpart) z) (and (and (= (. (TOK_TABLE_OR_COL subq) key1) (. (TOK_TABLE_OR_COL z) key)) (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08')) (= (. (TOK_TABLE_OR_COL z) hr) 11)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST z))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq) key1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage
+ Stage-7 depends on stages: Stage-8
+ Stage-0 depends on stages: Stage-7
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ subq:x
+ Fetch Operator
+ limit: -1
+ z
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ subq:x
+ TableScan
+ alias: x
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+ z
+ TableScan
+ alias: z
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq:y
+ TableScan
+ alias: y
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+150 val_150
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+66 val_66
+98 val_98
+98 val_98
+98 val_98
+98 val_98
Index: ql/src/test/results/clientpositive/bucketcontext_8.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_8.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_8.q.out (working copy)
@@ -327,41 +327,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -461,7 +460,169 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5500
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5500
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_big/ds=2008-04-09 [b]
+ /bucket_small/ds=2008-04-08 [a]
+ /bucket_small/ds=2008-04-09 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -491,8 +652,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
- /bucket_big/ds=2008-04-09 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/bucketcontext_3.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_3.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_3.q.out (working copy)
@@ -251,41 +251,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -337,7 +336,168 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5500
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5500
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_small/ds=2008-04-08 [a]
+ /bucket_small/ds=2008-04-09 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -367,7 +527,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/bucketmapjoin12.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketmapjoin12.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketmapjoin12.q.out (working copy)
@@ -74,14 +74,14 @@
POSTHOOK: type: ALTERTABLE_CLUSTER_SORT
POSTHOOK: Input: default@srcbucket_mapjoin_part_3
POSTHOOK: Output: default@srcbucket_mapjoin_part_3
-PREHOOK: query: -- The partition bucketing metadata match but one table is not bucketed, bucket map join should still be used
+PREHOOK: query: -- Dont ignore the mapjoin hint. Look at HIVE-4042 for details.
EXPLAIN EXTENDED
SELECT /*+ MAPJOIN(b) */ count(*)
FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2 b
ON a.key = b.key AND a.part = '1' and b.part = '1'
PREHOOK: type: QUERY
-POSTHOOK: query: -- The partition bucketing metadata match but one table is not bucketed, bucket map join should still be used
+POSTHOOK: query: -- Dont ignore the mapjoin hint. Look at HIVE-4042 for details.
EXPLAIN EXTENDED
SELECT /*+ MAPJOIN(b) */ count(*)
Index: ql/src/test/results/clientpositive/union22_2.q.out
===================================================================
--- ql/src/test/results/clientpositive/union22_2.q.out (revision 0)
+++ ql/src/test/results/clientpositive/union22_2.q.out (working copy)
@@ -0,0 +1,1796 @@
+PREHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dst_union22
+PREHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dst_union22_delta
+PREHOOK: query: insert overwrite table dst_union22 partition (ds='1')
+select key, value, key , value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dst_union22@ds=1
+POSTHOOK: query: insert overwrite table dst_union22 partition (ds='1')
+select key, value, key , value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dst_union22@ds=1
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table dst_union22_delta partition (ds='1')
+select key, key, value, key, value, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dst_union22_delta@ds=1
+POSTHOOK: query: insert overwrite table dst_union22_delta partition (ds='1')
+select key, key, value, key, value, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dst_union22_delta@ds=1
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- Union followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+explain extended
+insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Union followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+explain extended
+insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dst_union22_delta))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL k1) k1) (TOK_SELEXPR (TOK_TABLE_OR_COL k2) k2) (TOK_SELEXPR (TOK_TABLE_OR_COL k3) k3) (TOK_SELEXPR (TOK_TABLE_OR_COL k4) k4)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (<= (TOK_TABLE_OR_COL k0) 50))))) (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME dst_union22) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dst_union22_delta))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (> (TOK_TABLE_OR_COL k0) 50))))) b) (and (= (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL b) k1)) (= (. (TOK_TABLE_OR_COL a) ds) '1')))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) k1) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) k2) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) k3) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) k4) k4)) (TOK_WHERE (> (. (TOK_TABLE_OR_COL a) k1) 20))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dst_union22) (TOK_PARTSPEC (TOK_PARTVAL ds '2')))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-7 is a root stage , consists of Stage-8, Stage-1
+ Stage-8 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-8
+ Stage-2 depends on stages: Stage-1, Stage-6
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ null-subquery2:subq-subquery2:b:dst_union22_delta
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ null-subquery2:subq-subquery2:b:dst_union22_delta
+ TableScan
+ alias: dst_union22_delta
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((k0 > 50.0) and (k1 > 20.0))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: k1
+ type: string
+ expr: k3
+ type: string
+ expr: k4
+ type: string
+ outputColumnNames: _col1, _col3, _col4
+ HashTable Sink Operator
+ condition expressions:
+ 0 {k1} {k2}
+ 1 {_col3} {_col4}
+ filter mappings:
+ 0 [1, 1]
+ filter predicates:
+ 0 {(ds = '1')}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[k1]]
+ 1 [Column[_col1]]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ null-subquery2:subq-subquery2:a
+ TableScan
+ alias: a
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: (k1 > 20.0)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {k1} {k2}
+ 1 {_col3} {_col4}
+ filter mappings:
+ 0 [1, 1]
+ filter predicates:
+ 0 {(ds = '1')}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[k1]]
+ 1 [Column[_col1]]
+ outputColumnNames: _col0, _col1, _col10, _col11
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col10
+ type: string
+ expr: _col11
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string,string,string,string
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Local Work:
+ Map Reduce Local Work
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 1
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22
+ name: default.dst_union22
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 1
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22_delta
+ name: default.dst_union22_delta
+ Truncated Path -> Alias:
+ /dst_union22/ds=1 [null-subquery2:subq-subquery2:a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ TableScan
+ GatherStats: false
+ Union
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2/
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ null-subquery1:subq-subquery1:dst_union22_delta
+ TableScan
+ alias: dst_union22_delta
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: (k0 <= 50.0)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: k1
+ type: string
+ expr: k2
+ type: string
+ expr: k3
+ type: string
+ expr: k4
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Union
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2/
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string,string,string,string
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string,string,string,string
+ escape.delim \
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 1
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22_delta
+ name: default.dst_union22_delta
+ Truncated Path -> Alias:
+ /dst_union22_delta/ds=1 [null-subquery1:subq-subquery1:dst_union22_delta]
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 2
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22
+#### A masked pattern was here ####
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ null-subquery2:subq-subquery2:a
+ TableScan
+ alias: a
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: (k1 > 20.0)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: k1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: k1
+ type: string
+ tag: 0
+ value expressions:
+ expr: k1
+ type: string
+ expr: k2
+ type: string
+ expr: ds
+ type: string
+ null-subquery2:subq-subquery2:b:dst_union22_delta
+ TableScan
+ alias: dst_union22_delta
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate:
+ expr: ((k0 > 50.0) and (k1 > 20.0))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: k1
+ type: string
+ expr: k3
+ type: string
+ expr: k4
+ type: string
+ outputColumnNames: _col1, _col3, _col4
+ Reduce Output Operator
+ key expressions:
+ expr: _col1
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col1
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 1
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k1,k2,k3,k4
+ columns.types string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 11124
+ serialization.ddl struct dst_union22 { string k1, string k2, string k3, string k4}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22
+ name: default.dst_union22
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 1
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns k0,k1,k2,k3,k4,k5
+ columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+ name default.dst_union22_delta
+ numFiles 1
+ numPartitions 1
+ numRows 500
+ partition_columns ds
+ rawDataSize 16936
+ serialization.ddl struct dst_union22_delta { string k0, string k1, string k2, string k3, string k4, string k5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 17436
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dst_union22_delta
+ name: default.dst_union22_delta
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col3} {VALUE._col4}
+ filter mappings:
+ 0 [1, 1]
+ filter predicates:
+ 0 {(VALUE._col4 = '1')}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col10, _col11
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col10
+ type: string
+ expr: _col11
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string,string,string,string
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /dst_union22/ds=1 [null-subquery2:subq-subquery2:a]
+ /dst_union22_delta/ds=1 [null-subquery2:subq-subquery2:b:dst_union22_delta]
+
+
+PREHOOK: query: insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dst_union22
+PREHOOK: Input: default@dst_union22@ds=1
+PREHOOK: Input: default@dst_union22_delta
+PREHOOK: Input: default@dst_union22_delta@ds=1
+PREHOOK: Output: default@dst_union22@ds=2
+POSTHOOK: query: insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dst_union22
+POSTHOOK: Input: default@dst_union22@ds=1
+POSTHOOK: Input: default@dst_union22_delta
+POSTHOOK: Input: default@dst_union22_delta@ds=1
+POSTHOOK: Output: default@dst_union22@ds=2
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from dst_union22 where ds = '2' order by k1, k2, k3, k4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dst_union22
+PREHOOK: Input: default@dst_union22@ds=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dst_union22 where ds = '2' order by k1, k2, k3, k4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dst_union22
+POSTHOOK: Input: default@dst_union22@ds=2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k1 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k1, type:string, comment:null), (dst_union22)a.FieldSchema(name:k1, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k2 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k2, type:string, comment:null), (dst_union22)a.FieldSchema(name:k2, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k3 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k3, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22 PARTITION(ds=2).k4 EXPRESSION [(dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), (dst_union22_delta)dst_union22_delta.FieldSchema(name:k4, type:string, comment:null), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0 0 val_0 2
+0 val_0 0 val_0 2
+0 val_0 0 val_0 2
+10 val_10 10 val_10 2
+100 val_100 100 val_100 2
+100 val_100 100 val_100 2
+100 val_100 100 val_100 2
+100 val_100 100 val_100 2
+103 val_103 103 val_103 2
+103 val_103 103 val_103 2
+103 val_103 103 val_103 2
+103 val_103 103 val_103 2
+104 val_104 104 val_104 2
+104 val_104 104 val_104 2
+104 val_104 104 val_104 2
+104 val_104 104 val_104 2
+105 val_105 105 val_105 2
+11 val_11 11 val_11 2
+111 val_111 111 val_111 2
+113 val_113 113 val_113 2
+113 val_113 113 val_113 2
+113 val_113 113 val_113 2
+113 val_113 113 val_113 2
+114 val_114 114 val_114 2
+116 val_116 116 val_116 2
+118 val_118 118 val_118 2
+118 val_118 118 val_118 2
+118 val_118 118 val_118 2
+118 val_118 118 val_118 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+119 val_119 119 val_119 2
+12 val_12 12 val_12 2
+12 val_12 12 val_12 2
+120 val_120 120 val_120 2
+120 val_120 120 val_120 2
+120 val_120 120 val_120 2
+120 val_120 120 val_120 2
+125 val_125 125 val_125 2
+125 val_125 125 val_125 2
+125 val_125 125 val_125 2
+125 val_125 125 val_125 2
+126 val_126 126 val_126 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+128 val_128 128 val_128 2
+129 val_129 129 val_129 2
+129 val_129 129 val_129 2
+129 val_129 129 val_129 2
+129 val_129 129 val_129 2
+131 val_131 131 val_131 2
+133 val_133 133 val_133 2
+134 val_134 134 val_134 2
+134 val_134 134 val_134 2
+134 val_134 134 val_134 2
+134 val_134 134 val_134 2
+136 val_136 136 val_136 2
+137 val_137 137 val_137 2
+137 val_137 137 val_137 2
+137 val_137 137 val_137 2
+137 val_137 137 val_137 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+138 val_138 138 val_138 2
+143 val_143 143 val_143 2
+145 val_145 145 val_145 2
+146 val_146 146 val_146 2
+146 val_146 146 val_146 2
+146 val_146 146 val_146 2
+146 val_146 146 val_146 2
+149 val_149 149 val_149 2
+149 val_149 149 val_149 2
+149 val_149 149 val_149 2
+149 val_149 149 val_149 2
+15 val_15 15 val_15 2
+15 val_15 15 val_15 2
+150 val_150 150 val_150 2
+152 val_152 152 val_152 2
+152 val_152 152 val_152 2
+152 val_152 152 val_152 2
+152 val_152 152 val_152 2
+153 val_153 153 val_153 2
+155 val_155 155 val_155 2
+156 val_156 156 val_156 2
+157 val_157 157 val_157 2
+158 val_158 158 val_158 2
+160 val_160 160 val_160 2
+162 val_162 162 val_162 2
+163 val_163 163 val_163 2
+164 val_164 164 val_164 2
+164 val_164 164 val_164 2
+164 val_164 164 val_164 2
+164 val_164 164 val_164 2
+165 val_165 165 val_165 2
+165 val_165 165 val_165 2
+165 val_165 165 val_165 2
+165 val_165 165 val_165 2
+166 val_166 166 val_166 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+167 val_167 167 val_167 2
+168 val_168 168 val_168 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+169 val_169 169 val_169 2
+17 val_17 17 val_17 2
+170 val_170 170 val_170 2
+172 val_172 172 val_172 2
+172 val_172 172 val_172 2
+172 val_172 172 val_172 2
+172 val_172 172 val_172 2
+174 val_174 174 val_174 2
+174 val_174 174 val_174 2
+174 val_174 174 val_174 2
+174 val_174 174 val_174 2
+175 val_175 175 val_175 2
+175 val_175 175 val_175 2
+175 val_175 175 val_175 2
+175 val_175 175 val_175 2
+176 val_176 176 val_176 2
+176 val_176 176 val_176 2
+176 val_176 176 val_176 2
+176 val_176 176 val_176 2
+177 val_177 177 val_177 2
+178 val_178 178 val_178 2
+179 val_179 179 val_179 2
+179 val_179 179 val_179 2
+179 val_179 179 val_179 2
+179 val_179 179 val_179 2
+18 val_18 18 val_18 2
+18 val_18 18 val_18 2
+180 val_180 180 val_180 2
+181 val_181 181 val_181 2
+183 val_183 183 val_183 2
+186 val_186 186 val_186 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+187 val_187 187 val_187 2
+189 val_189 189 val_189 2
+19 val_19 19 val_19 2
+190 val_190 190 val_190 2
+191 val_191 191 val_191 2
+191 val_191 191 val_191 2
+191 val_191 191 val_191 2
+191 val_191 191 val_191 2
+192 val_192 192 val_192 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+193 val_193 193 val_193 2
+194 val_194 194 val_194 2
+195 val_195 195 val_195 2
+195 val_195 195 val_195 2
+195 val_195 195 val_195 2
+195 val_195 195 val_195 2
+196 val_196 196 val_196 2
+197 val_197 197 val_197 2
+197 val_197 197 val_197 2
+197 val_197 197 val_197 2
+197 val_197 197 val_197 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+199 val_199 199 val_199 2
+2 val_2 2 val_2 2
+20 val_20 20 val_20 2
+200 val_200 200 val_200 2
+200 val_200 200 val_200 2
+200 val_200 200 val_200 2
+200 val_200 200 val_200 2
+201 val_201 201 val_201 2
+202 val_202 202 val_202 2
+203 val_203 203 val_203 2
+203 val_203 203 val_203 2
+203 val_203 203 val_203 2
+203 val_203 203 val_203 2
+205 val_205 205 val_205 2
+205 val_205 205 val_205 2
+205 val_205 205 val_205 2
+205 val_205 205 val_205 2
+207 val_207 207 val_207 2
+207 val_207 207 val_207 2
+207 val_207 207 val_207 2
+207 val_207 207 val_207 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+208 val_208 208 val_208 2
+209 val_209 209 val_209 2
+209 val_209 209 val_209 2
+209 val_209 209 val_209 2
+209 val_209 209 val_209 2
+213 val_213 213 val_213 2
+213 val_213 213 val_213 2
+213 val_213 213 val_213 2
+213 val_213 213 val_213 2
+214 val_214 214 val_214 2
+216 val_216 216 val_216 2
+216 val_216 216 val_216 2
+216 val_216 216 val_216 2
+216 val_216 216 val_216 2
+217 val_217 217 val_217 2
+217 val_217 217 val_217 2
+217 val_217 217 val_217 2
+217 val_217 217 val_217 2
+218 val_218 218 val_218 2
+219 val_219 219 val_219 2
+219 val_219 219 val_219 2
+219 val_219 219 val_219 2
+219 val_219 219 val_219 2
+221 val_221 221 val_221 2
+221 val_221 221 val_221 2
+221 val_221 221 val_221 2
+221 val_221 221 val_221 2
+222 val_222 222 val_222 2
+223 val_223 223 val_223 2
+223 val_223 223 val_223 2
+223 val_223 223 val_223 2
+223 val_223 223 val_223 2
+224 val_224 224 val_224 2
+224 val_224 224 val_224 2
+224 val_224 224 val_224 2
+224 val_224 224 val_224 2
+226 val_226 226 val_226 2
+228 val_228 228 val_228 2
+229 val_229 229 val_229 2
+229 val_229 229 val_229 2
+229 val_229 229 val_229 2
+229 val_229 229 val_229 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+230 val_230 230 val_230 2
+233 val_233 233 val_233 2
+233 val_233 233 val_233 2
+233 val_233 233 val_233 2
+233 val_233 233 val_233 2
+235 val_235 235 val_235 2
+237 val_237 237 val_237 2
+237 val_237 237 val_237 2
+237 val_237 237 val_237 2
+237 val_237 237 val_237 2
+238 val_238 238 val_238 2
+238 val_238 238 val_238 2
+238 val_238 238 val_238 2
+238 val_238 238 val_238 2
+239 val_239 239 val_239 2
+239 val_239 239 val_239 2
+239 val_239 239 val_239 2
+239 val_239 239 val_239 2
+24 val_24 NULL NULL 2
+24 val_24 NULL NULL 2
+24 val_24 24 val_24 2
+24 val_24 24 val_24 2
+241 val_241 241 val_241 2
+242 val_242 242 val_242 2
+242 val_242 242 val_242 2
+242 val_242 242 val_242 2
+242 val_242 242 val_242 2
+244 val_244 244 val_244 2
+247 val_247 247 val_247 2
+248 val_248 248 val_248 2
+249 val_249 249 val_249 2
+252 val_252 252 val_252 2
+255 val_255 255 val_255 2
+255 val_255 255 val_255 2
+255 val_255 255 val_255 2
+255 val_255 255 val_255 2
+256 val_256 256 val_256 2
+256 val_256 256 val_256 2
+256 val_256 256 val_256 2
+256 val_256 256 val_256 2
+257 val_257 257 val_257 2
+258 val_258 258 val_258 2
+26 val_26 NULL NULL 2
+26 val_26 NULL NULL 2
+26 val_26 26 val_26 2
+26 val_26 26 val_26 2
+260 val_260 260 val_260 2
+262 val_262 262 val_262 2
+263 val_263 263 val_263 2
+265 val_265 265 val_265 2
+265 val_265 265 val_265 2
+265 val_265 265 val_265 2
+265 val_265 265 val_265 2
+266 val_266 266 val_266 2
+27 val_27 NULL NULL 2
+27 val_27 27 val_27 2
+272 val_272 272 val_272 2
+272 val_272 272 val_272 2
+272 val_272 272 val_272 2
+272 val_272 272 val_272 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+273 val_273 273 val_273 2
+274 val_274 274 val_274 2
+275 val_275 275 val_275 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+277 val_277 277 val_277 2
+278 val_278 278 val_278 2
+278 val_278 278 val_278 2
+278 val_278 278 val_278 2
+278 val_278 278 val_278 2
+28 val_28 NULL NULL 2
+28 val_28 28 val_28 2
+280 val_280 280 val_280 2
+280 val_280 280 val_280 2
+280 val_280 280 val_280 2
+280 val_280 280 val_280 2
+281 val_281 281 val_281 2
+281 val_281 281 val_281 2
+281 val_281 281 val_281 2
+281 val_281 281 val_281 2
+282 val_282 282 val_282 2
+282 val_282 282 val_282 2
+282 val_282 282 val_282 2
+282 val_282 282 val_282 2
+283 val_283 283 val_283 2
+284 val_284 284 val_284 2
+285 val_285 285 val_285 2
+286 val_286 286 val_286 2
+287 val_287 287 val_287 2
+288 val_288 288 val_288 2
+288 val_288 288 val_288 2
+288 val_288 288 val_288 2
+288 val_288 288 val_288 2
+289 val_289 289 val_289 2
+291 val_291 291 val_291 2
+292 val_292 292 val_292 2
+296 val_296 296 val_296 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+298 val_298 298 val_298 2
+30 val_30 NULL NULL 2
+30 val_30 30 val_30 2
+302 val_302 302 val_302 2
+305 val_305 305 val_305 2
+306 val_306 306 val_306 2
+307 val_307 307 val_307 2
+307 val_307 307 val_307 2
+307 val_307 307 val_307 2
+307 val_307 307 val_307 2
+308 val_308 308 val_308 2
+309 val_309 309 val_309 2
+309 val_309 309 val_309 2
+309 val_309 309 val_309 2
+309 val_309 309 val_309 2
+310 val_310 310 val_310 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+311 val_311 311 val_311 2
+315 val_315 315 val_315 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+316 val_316 316 val_316 2
+317 val_317 317 val_317 2
+317 val_317 317 val_317 2
+317 val_317 317 val_317 2
+317 val_317 317 val_317 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+318 val_318 318 val_318 2
+321 val_321 321 val_321 2
+321 val_321 321 val_321 2
+321 val_321 321 val_321 2
+321 val_321 321 val_321 2
+322 val_322 322 val_322 2
+322 val_322 322 val_322 2
+322 val_322 322 val_322 2
+322 val_322 322 val_322 2
+323 val_323 323 val_323 2
+325 val_325 325 val_325 2
+325 val_325 325 val_325 2
+325 val_325 325 val_325 2
+325 val_325 325 val_325 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+327 val_327 327 val_327 2
+33 val_33 NULL NULL 2
+33 val_33 33 val_33 2
+331 val_331 331 val_331 2
+331 val_331 331 val_331 2
+331 val_331 331 val_331 2
+331 val_331 331 val_331 2
+332 val_332 332 val_332 2
+333 val_333 333 val_333 2
+333 val_333 333 val_333 2
+333 val_333 333 val_333 2
+333 val_333 333 val_333 2
+335 val_335 335 val_335 2
+336 val_336 336 val_336 2
+338 val_338 338 val_338 2
+339 val_339 339 val_339 2
+34 val_34 NULL NULL 2
+34 val_34 34 val_34 2
+341 val_341 341 val_341 2
+342 val_342 342 val_342 2
+342 val_342 342 val_342 2
+342 val_342 342 val_342 2
+342 val_342 342 val_342 2
+344 val_344 344 val_344 2
+344 val_344 344 val_344 2
+344 val_344 344 val_344 2
+344 val_344 344 val_344 2
+345 val_345 345 val_345 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+348 val_348 348 val_348 2
+35 val_35 NULL NULL 2
+35 val_35 NULL NULL 2
+35 val_35 NULL NULL 2
+35 val_35 35 val_35 2
+35 val_35 35 val_35 2
+35 val_35 35 val_35 2
+351 val_351 351 val_351 2
+353 val_353 353 val_353 2
+353 val_353 353 val_353 2
+353 val_353 353 val_353 2
+353 val_353 353 val_353 2
+356 val_356 356 val_356 2
+360 val_360 360 val_360 2
+362 val_362 362 val_362 2
+364 val_364 364 val_364 2
+365 val_365 365 val_365 2
+366 val_366 366 val_366 2
+367 val_367 367 val_367 2
+367 val_367 367 val_367 2
+367 val_367 367 val_367 2
+367 val_367 367 val_367 2
+368 val_368 368 val_368 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+369 val_369 369 val_369 2
+37 val_37 NULL NULL 2
+37 val_37 NULL NULL 2
+37 val_37 37 val_37 2
+37 val_37 37 val_37 2
+373 val_373 373 val_373 2
+374 val_374 374 val_374 2
+375 val_375 375 val_375 2
+377 val_377 377 val_377 2
+378 val_378 378 val_378 2
+379 val_379 379 val_379 2
+382 val_382 382 val_382 2
+382 val_382 382 val_382 2
+382 val_382 382 val_382 2
+382 val_382 382 val_382 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+384 val_384 384 val_384 2
+386 val_386 386 val_386 2
+389 val_389 389 val_389 2
+392 val_392 392 val_392 2
+393 val_393 393 val_393 2
+394 val_394 394 val_394 2
+395 val_395 395 val_395 2
+395 val_395 395 val_395 2
+395 val_395 395 val_395 2
+395 val_395 395 val_395 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+396 val_396 396 val_396 2
+397 val_397 397 val_397 2
+397 val_397 397 val_397 2
+397 val_397 397 val_397 2
+397 val_397 397 val_397 2
+399 val_399 399 val_399 2
+399 val_399 399 val_399 2
+399 val_399 399 val_399 2
+399 val_399 399 val_399 2
+4 val_4 4 val_4 2
+400 val_400 400 val_400 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+401 val_401 401 val_401 2
+402 val_402 402 val_402 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+403 val_403 403 val_403 2
+404 val_404 404 val_404 2
+404 val_404 404 val_404 2
+404 val_404 404 val_404 2
+404 val_404 404 val_404 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+406 val_406 406 val_406 2
+407 val_407 407 val_407 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+409 val_409 409 val_409 2
+41 val_41 NULL NULL 2
+41 val_41 41 val_41 2
+411 val_411 411 val_411 2
+413 val_413 413 val_413 2
+413 val_413 413 val_413 2
+413 val_413 413 val_413 2
+413 val_413 413 val_413 2
+414 val_414 414 val_414 2
+414 val_414 414 val_414 2
+414 val_414 414 val_414 2
+414 val_414 414 val_414 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+417 val_417 417 val_417 2
+418 val_418 418 val_418 2
+419 val_419 419 val_419 2
+42 val_42 NULL NULL 2
+42 val_42 NULL NULL 2
+42 val_42 42 val_42 2
+42 val_42 42 val_42 2
+421 val_421 421 val_421 2
+424 val_424 424 val_424 2
+424 val_424 424 val_424 2
+424 val_424 424 val_424 2
+424 val_424 424 val_424 2
+427 val_427 427 val_427 2
+429 val_429 429 val_429 2
+429 val_429 429 val_429 2
+429 val_429 429 val_429 2
+429 val_429 429 val_429 2
+43 val_43 NULL NULL 2
+43 val_43 43 val_43 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+430 val_430 430 val_430 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+431 val_431 431 val_431 2
+432 val_432 432 val_432 2
+435 val_435 435 val_435 2
+436 val_436 436 val_436 2
+437 val_437 437 val_437 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+438 val_438 438 val_438 2
+439 val_439 439 val_439 2
+439 val_439 439 val_439 2
+439 val_439 439 val_439 2
+439 val_439 439 val_439 2
+44 val_44 NULL NULL 2
+44 val_44 44 val_44 2
+443 val_443 443 val_443 2
+444 val_444 444 val_444 2
+446 val_446 446 val_446 2
+448 val_448 448 val_448 2
+449 val_449 449 val_449 2
+452 val_452 452 val_452 2
+453 val_453 453 val_453 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+454 val_454 454 val_454 2
+455 val_455 455 val_455 2
+457 val_457 457 val_457 2
+458 val_458 458 val_458 2
+458 val_458 458 val_458 2
+458 val_458 458 val_458 2
+458 val_458 458 val_458 2
+459 val_459 459 val_459 2
+459 val_459 459 val_459 2
+459 val_459 459 val_459 2
+459 val_459 459 val_459 2
+460 val_460 460 val_460 2
+462 val_462 462 val_462 2
+462 val_462 462 val_462 2
+462 val_462 462 val_462 2
+462 val_462 462 val_462 2
+463 val_463 463 val_463 2
+463 val_463 463 val_463 2
+463 val_463 463 val_463 2
+463 val_463 463 val_463 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+466 val_466 466 val_466 2
+467 val_467 467 val_467 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+468 val_468 468 val_468 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+469 val_469 469 val_469 2
+47 val_47 NULL NULL 2
+47 val_47 47 val_47 2
+470 val_470 470 val_470 2
+472 val_472 472 val_472 2
+475 val_475 475 val_475 2
+477 val_477 477 val_477 2
+478 val_478 478 val_478 2
+478 val_478 478 val_478 2
+478 val_478 478 val_478 2
+478 val_478 478 val_478 2
+479 val_479 479 val_479 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+480 val_480 480 val_480 2
+481 val_481 481 val_481 2
+482 val_482 482 val_482 2
+483 val_483 483 val_483 2
+484 val_484 484 val_484 2
+485 val_485 485 val_485 2
+487 val_487 487 val_487 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+489 val_489 489 val_489 2
+490 val_490 490 val_490 2
+491 val_491 491 val_491 2
+492 val_492 492 val_492 2
+492 val_492 492 val_492 2
+492 val_492 492 val_492 2
+492 val_492 492 val_492 2
+493 val_493 493 val_493 2
+494 val_494 494 val_494 2
+495 val_495 495 val_495 2
+496 val_496 496 val_496 2
+497 val_497 497 val_497 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+498 val_498 498 val_498 2
+5 val_5 5 val_5 2
+5 val_5 5 val_5 2
+5 val_5 5 val_5 2
+51 val_51 51 val_51 2
+51 val_51 51 val_51 2
+51 val_51 51 val_51 2
+51 val_51 51 val_51 2
+53 val_53 53 val_53 2
+54 val_54 54 val_54 2
+57 val_57 57 val_57 2
+58 val_58 58 val_58 2
+58 val_58 58 val_58 2
+58 val_58 58 val_58 2
+58 val_58 58 val_58 2
+64 val_64 64 val_64 2
+65 val_65 65 val_65 2
+66 val_66 66 val_66 2
+67 val_67 67 val_67 2
+67 val_67 67 val_67 2
+67 val_67 67 val_67 2
+67 val_67 67 val_67 2
+69 val_69 69 val_69 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+70 val_70 70 val_70 2
+72 val_72 72 val_72 2
+72 val_72 72 val_72 2
+72 val_72 72 val_72 2
+72 val_72 72 val_72 2
+74 val_74 74 val_74 2
+76 val_76 76 val_76 2
+76 val_76 76 val_76 2
+76 val_76 76 val_76 2
+76 val_76 76 val_76 2
+77 val_77 77 val_77 2
+78 val_78 78 val_78 2
+8 val_8 8 val_8 2
+80 val_80 80 val_80 2
+82 val_82 82 val_82 2
+83 val_83 83 val_83 2
+83 val_83 83 val_83 2
+83 val_83 83 val_83 2
+83 val_83 83 val_83 2
+84 val_84 84 val_84 2
+84 val_84 84 val_84 2
+84 val_84 84 val_84 2
+84 val_84 84 val_84 2
+85 val_85 85 val_85 2
+86 val_86 86 val_86 2
+87 val_87 87 val_87 2
+9 val_9 9 val_9 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+90 val_90 90 val_90 2
+92 val_92 92 val_92 2
+95 val_95 95 val_95 2
+95 val_95 95 val_95 2
+95 val_95 95 val_95 2
+95 val_95 95 val_95 2
+96 val_96 96 val_96 2
+97 val_97 97 val_97 2
+97 val_97 97 val_97 2
+97 val_97 97 val_97 2
+97 val_97 97 val_97 2
+98 val_98 98 val_98 2
+98 val_98 98 val_98 2
+98 val_98 98 val_98 2
+98 val_98 98 val_98 2
Index: ql/src/test/results/clientpositive/bucketcontext_7.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_7.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_7.q.out (working copy)
@@ -327,41 +327,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -461,7 +460,169 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 8
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-09
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-09
+ properties:
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 8
+ numPartitions 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 11624
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_big/ds=2008-04-09 [b]
+ /bucket_small/ds=2008-04-08 [a]
+ /bucket_small/ds=2008-04-09 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -491,8 +652,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
- /bucket_big/ds=2008-04-09 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/bucketcontext_2.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_2.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_2.q.out (working copy)
@@ -300,41 +300,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -434,7 +433,120 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 1
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_big/ds=2008-04-09 [b]
+ /bucket_small/ds=2008-04-08 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -464,8 +576,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
- /bucket_big/ds=2008-04-09 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/join32.q.out
===================================================================
--- ql/src/test/results/clientpositive/join32.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/join32.q.out (working copy)
@@ -369,3 +369,388 @@
98 val_98 val_98
98 val_98 val_98
98 val_98 val_98
+PREHOOK: query: -- Mapjoin followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Mapjoin followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key))) (TOK_TABREF (TOK_TABNAME srcpart) z) (and (and (= (. (TOK_TABLE_OR_COL x) value) (. (TOK_TABLE_OR_COL z) value)) (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08')) (= (. (TOK_TABLE_OR_COL z) hr) 11)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest_j1))) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x z))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL y) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-7 is a root stage
+ Stage-6 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-6
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ x
+ Fetch Operator
+ limit: -1
+ z
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ x
+ TableScan
+ alias: x
+ GatherStats: false
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+ z
+ TableScan
+ alias: z
+ GatherStats: false
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col5} {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col1]]
+ 1 [Column[value]]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ y
+ TableScan
+ alias: y
+ GatherStats: false
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col5
+ Position of Big Table: 1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col5} {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col1]]
+ 1 [Column[value]]
+ outputColumnNames: _col1, _col4, _col9
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col9
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:string
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 1
+ numPartitions 0
+ numRows 85
+ rawDataSize 1600
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 1685
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Local Work:
+ Map Reduce Local Work
+ Needs Tagging: true
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+#### A masked pattern was here ####
+ Partition
+ base file name: src1
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src1
+ numFiles 1
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct src1 { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 216
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src1
+ name: default.src1
+ Truncated Path -> Alias:
+ /src [y]
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value,val2
+ columns.types string:string:string
+#### A masked pattern was here ####
+ name default.dest_j1
+ numFiles 1
+ numPartitions 0
+ numRows 85
+ rawDataSize 1600
+ serialization.ddl struct dest_j1 { string key, string value, string val2}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 1685
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
+#### A masked pattern was here ####
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+146 val_146 val_146
+146 val_146 val_146
+146 val_146 val_146
+146 val_146 val_146
+150 val_150 val_150
+213 val_213 val_213
+213 val_213 val_213
+213 val_213 val_213
+213 val_213 val_213
+238 val_238 val_238
+238 val_238 val_238
+238 val_238 val_238
+238 val_238 val_238
+255 val_255 val_255
+255 val_255 val_255
+255 val_255 val_255
+255 val_255 val_255
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+273 val_273 val_273
+278 val_278 val_278
+278 val_278 val_278
+278 val_278 val_278
+278 val_278 val_278
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+311 val_311 val_311
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+401 val_401 val_401
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+406 val_406 val_406
+66 val_66 val_66
+98 val_98 val_98
+98 val_98 val_98
+98 val_98 val_98
+98 val_98 val_98
Index: ql/src/test/results/clientpositive/bucketcontext_6.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_6.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_6.q.out (working copy)
@@ -297,41 +297,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -431,7 +430,118 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: bucket_small
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_big/ds=2008-04-09 [b]
+ /bucket_small [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -461,8 +571,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
- /bucket_big/ds=2008-04-09 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/bucketcontext_1.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_1.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_1.q.out (working copy)
@@ -312,41 +312,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -446,7 +445,120 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: ds=2008-04-08
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ properties:
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 2
+ numPartitions 1
+ numRows 0
+ partition_columns ds
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 2750
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big/ds=2008-04-08 [b]
+ /bucket_big/ds=2008-04-09 [b]
+ /bucket_small/ds=2008-04-08 [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -476,8 +588,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [b]
- /bucket_big/ds=2008-04-09 [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/results/clientpositive/smb_mapjoin_14_2.q.out
===================================================================
--- ql/src/test/results/clientpositive/smb_mapjoin_14_2.q.out (revision 0)
+++ ql/src/test/results/clientpositive/smb_mapjoin_14_2.q.out (working copy)
@@ -0,0 +1,2015 @@
+PREHOOK: query: CREATE TABLE tbl1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE tbl1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tbl1
+PREHOOK: query: CREATE TABLE tbl2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE tbl2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tbl2
+PREHOOK: query: insert overwrite table tbl1
+select * from src where key < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tbl1
+POSTHOOK: query: insert overwrite table tbl1
+select * from src where key < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tbl1
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tbl2
+select * from src where key < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tbl2
+POSTHOOK: query: insert overwrite table tbl2
+select * from src where key < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tbl2
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- All the mapjoin hints would be ignored
+-- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+PREHOOK: type: QUERY
+POSTHOOK: query: -- All the mapjoin hints would be ignored
+-- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) val1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) val2)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 0
+ subq1:b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+22
+PREHOOK: query: -- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+-- Add a order by at the end to make the results deterministic.
+explain
+select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+-- Add a order by at the end to make the results deterministic.
+explain
+select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) val1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) val2)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONSTAR count))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 0
+ value expressions:
+ expr: key
+ type: int
+ subq1:b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: int
+ mode: hash
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: int
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: int
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 9
+2 1
+4 1
+5 9
+8 1
+9 1
+PREHOOK: query: -- The mapjoin is being performed as part of more than one sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The mapjoin is being performed as part of more than one sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME tbl1) a) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) val1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) val2)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONSTAR count))) (TOK_GROUPBY (TOK_TABLE_OR_COL key)))) subq2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq2:subq1:a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 0
+ value expressions:
+ expr: key
+ type: int
+ subq2:subq1:b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: int
+ mode: hash
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: int
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+6
+PREHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl2) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq2) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ subq2:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join, although there is more than one level of sub-query
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join, although there is more than one level of sub-query
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq2) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL subq2) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq2))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ subq2:subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key < 8) and (key < 6))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- Both the big table and the small table are nested sub-queries i.e more then 1 level of sub-query.
+-- The join should be converted to a sort-merge join
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Both the big table and the small table are nested sub-queries i.e more then 1 level of sub-query.
+-- The join should be converted to a sort-merge join
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq2) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq4) (= (. (TOK_TABLE_OR_COL subq2) key) (. (TOK_TABLE_OR_COL subq4) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq2))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq2:subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key < 8) and (key < 6))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ subq4:subq3:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key < 8) and (key < 6))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters and the join key
+-- is not getting modified, it should be converted to a sort-merge join. Note that the sub-query modifies one
+-- item, but that is not part of the join key.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The subquery itself is being map-joined. Since the sub-query only contains selects and filters and the join key
+-- is not getting modified, it should be converted to a sort-merge join. Note that the sub-query modifies one
+-- item, but that is not part of the join key.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL a) value)) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl2) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL a) value)) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq2) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 8)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ subq2:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 8)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- Since the join key is modified by the sub-query, neither sort-merge join not bucketized map-side
+-- join should be performed
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Since the join key is modified by the sub-query, neither sort-merge join not bucketized map-side
+-- join should be performed
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (. (TOK_TABLE_OR_COL a) key) 1) key) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL a) value)) value)))) subq1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl2) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (. (TOK_TABLE_OR_COL a) key) 1) key) (TOK_SELEXPR (TOK_FUNCTION concat (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL a) value)) value)))) subq2) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Select Operator
+ expressions:
+ expr: (key + 1)
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ subq2:a
+ TableScan
+ alias: a
+ Select Operator
+ expressions:
+ expr: (key + 1)
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+22
+PREHOOK: query: -- The small table is a sub-query and the big table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The small table is a sub-query and the big table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq1) (TOK_TABREF (TOK_TABNAME tbl2) a) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL a) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- The big table is a sub-query and the small table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The big table is a sub-query and the small table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq1) (TOK_TABREF (TOK_TABNAME tbl2) a) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL a) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST a))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
+PREHOOK: query: -- There are more than 2 inputs to the join, all of them being sub-queries.
+-- It should be converted to to a sort-merge join
+explain
+select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on (subq1.key = subq2.key)
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: -- There are more than 2 inputs to the join, all of them being sub-queries.
+-- It should be converted to to a sort-merge join
+explain
+select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on (subq1.key = subq2.key)
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl2) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq2) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq2) key))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl2) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq3) (= (. (TOK_TABLE_OR_COL subq1) key) (. (TOK_TABLE_OR_COL subq3) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq1 subq2))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ subq2:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 1
+ subq3:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: (key < 6)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0
+ 1
+ 2
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+56
+PREHOOK: query: -- The mapjoin is being performed on a nested sub-query, and an aggregation is performed after that.
+-- The join should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a
+PREHOOK: type: QUERY
+POSTHOOK: query: -- The mapjoin is being performed on a nested sub-query, and an aggregation is performed after that.
+-- The join should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME tbl1) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value) value)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 8)))) subq1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (TOK_TABLE_OR_COL key) 6)))) subq2) (TOK_TABREF (TOK_TABNAME tbl2) b) (= (. (TOK_TABLE_OR_COL subq2) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST subq2))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq2) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL subq2) value) value1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) value2)))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: int
+ tag: 1
+ a:subq2:subq1:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key < 8) and (key < 6))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: int
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: int
+ tag: 0
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl1
+PREHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl1
+POSTHOOK: Input: default@tbl2
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tbl1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tbl2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+20
Index: ql/src/test/results/clientpositive/bucketcontext_5.q.out
===================================================================
--- ql/src/test/results/clientpositive/bucketcontext_5.q.out (revision 1459200)
+++ ql/src/test/results/clientpositive/bucketcontext_5.q.out (working copy)
@@ -228,41 +228,40 @@
STAGE DEPENDENCIES:
Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ GatherStats: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
b
TableScan
alias: b
GatherStats: false
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- Needs Tagging: false
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Needs Tagging: true
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -312,7 +311,117 @@
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.bucket_big
name: default.bucket_big
+#### A masked pattern was here ####
+ Partition
+ base file name: bucket_small
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 4
+ bucket_field_name key
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.bucket_small
+ numFiles 4
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ serialization.ddl struct bucket_small { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucket_small
+ name: default.bucket_small
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Truncated Path -> Alias:
+ /bucket_big [b]
+ /bucket_small [a]
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Needs Tagging: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10002
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ Reduce Operator Tree:
Group By Operator
aggregations:
expr: count(VALUE._col0)
@@ -342,7 +451,7 @@
GatherStats: false
MultiFileSpray: false
Truncated Path -> Alias:
- /bucket_big [b]
+#### A masked pattern was here ####
Stage: Stage-0
Fetch Operator
Index: ql/src/test/queries/clientpositive/select_transform_hint.q
===================================================================
--- ql/src/test/queries/clientpositive/select_transform_hint.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/select_transform_hint.q (working copy)
@@ -1,3 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
+
EXPLAIN
SELECT /*+MAPJOIN(a)*/
TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue)
Index: ql/src/test/queries/clientpositive/bucketmapjoin9.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin9.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin9.q (working copy)
@@ -14,6 +14,7 @@
ALTER TABLE srcbucket_mapjoin_part_2 CLUSTERED BY (key) INTO 2 BUCKETS;
set hive.optimize.bucketmapjoin=true;
+set hive.ignore.mapjoin.hint=false;
-- The table bucketing metadata matches but the partitions have different numbers of buckets, bucket map join should not be used
Index: ql/src/test/queries/clientpositive/bucketcontext_8.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_8.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_8.q (working copy)
@@ -18,9 +18,11 @@
load data local inpath '../data/files/srcsortbucket4outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
+set hive.ignore.mapjoin.hint=true;
set hive.optimize.bucketmapjoin.sortedmerge = true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/union22_2.q
===================================================================
--- ql/src/test/queries/clientpositive/union22_2.q (revision 0)
+++ ql/src/test/queries/clientpositive/union22_2.q (working copy)
@@ -0,0 +1,44 @@
+create table dst_union22(k1 string, k2 string, k3 string, k4 string) partitioned by (ds string);
+create table dst_union22_delta(k0 string, k1 string, k2 string, k3 string, k4 string, k5 string) partitioned by (ds string);
+
+insert overwrite table dst_union22 partition (ds='1')
+select key, value, key , value from src;
+
+insert overwrite table dst_union22_delta partition (ds='1')
+select key, key, value, key, value, value from src;
+
+set hive.merge.mapfiles=false;
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.ignore.mapjoin.hint = true;
+
+-- Union followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+explain extended
+insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq;
+
+insert overwrite table dst_union22 partition (ds='2')
+select * from
+(
+select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0 <= 50
+union all
+select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0 > 50) b on
+a.k1 = b.k1 and a.ds='1'
+where a.k1 > 20
+)
+subq;
+
+select * from dst_union22 where ds = '2' order by k1, k2, k3, k4;
Index: ql/src/test/queries/clientpositive/join32.q
===================================================================
--- ql/src/test/queries/clientpositive/join32.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join32.q (working copy)
@@ -19,5 +19,17 @@
select * from dest_j1 x order by x.key;
+-- Mapjoin followed by Mapjoin is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,z) */ x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
+select * from dest_j1 x order by x.key;
Index: ql/src/test/queries/clientpositive/join25.q
===================================================================
--- ql/src/test/queries/clientpositive/join25.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join25.q (working copy)
@@ -1,7 +1,7 @@
set hive.mapjoin.numrows = 2;
+set hive.ignore.mapjoin.hint=false;
-
CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE;
EXPLAIN
Index: ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q
===================================================================
--- ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/mapjoin_filter_on_outerjoin.q (working copy)
@@ -1,4 +1,5 @@
set hive.auto.convert.join = false;
+set hive.ignore.mapjoin.hint=false;
--HIVE-2101 mapjoin sometimes gives wrong results if there is a filter in the on condition
SELECT * FROM src1
Index: ql/src/test/queries/clientpositive/join36.q
===================================================================
--- ql/src/test/queries/clientpositive/join36.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join36.q (working copy)
@@ -1,9 +1,6 @@
set hive.mapjoin.numrows = 2;
+set hive.ignore.mapjoin.hint=false;
-
-
-
-
CREATE TABLE tmp1(key INT, cnt INT);
CREATE TABLE tmp2(key INT, cnt INT);
CREATE TABLE dest_j1(key INT, value INT, val2 INT);
@@ -24,6 +21,3 @@
FROM tmp1 x JOIN tmp2 y ON (x.key = y.key);
select * from dest_j1 x order by x.key;
-
-
-
Index: ql/src/test/queries/clientpositive/join29.q
===================================================================
--- ql/src/test/queries/clientpositive/join29.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join29.q (working copy)
@@ -18,3 +18,18 @@
(select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key);
select * from dest_j1 x order by x.key;
+
+-- Mapjoin followed by group by is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key);
+
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(subq1) */ subq1.key, subq1.cnt, subq2.cnt
+FROM (select x.key, count(1) as cnt from src1 x group by x.key) subq1 JOIN
+ (select y.key, count(1) as cnt from src y group by y.key) subq2 ON (subq1.key = subq2.key);
+
+select * from dest_j1 x order by x.key;
Index: ql/src/test/queries/clientpositive/bucketmapjoin_negative.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin_negative.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin_negative.q (working copy)
@@ -1,7 +1,3 @@
-
-
-
-
CREATE TABLE srcbucket_mapjoin(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin;
load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin;
@@ -11,8 +7,7 @@
load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
-
-
+set hive.ignore.mapjoin.hint=false;
set hive.optimize.bucketmapjoin = true;
create table bucketmapjoin_tmp_result (key string , value1 string, value2 string);
Index: ql/src/test/queries/clientpositive/bucketmapjoin2.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin2.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin2.q (working copy)
@@ -1,3 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
+
CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
Index: ql/src/test/queries/clientpositive/join_map_ppr.q
===================================================================
--- ql/src/test/queries/clientpositive/join_map_ppr.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join_map_ppr.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE;
EXPLAIN EXTENDED
@@ -34,8 +35,3 @@
WHERE z.ds='2008-04-08' and z.hr=11;
select * from dest_j1 x order by x.key;
-
-
-
-
-
Index: ql/src/test/queries/clientpositive/bucketcontext_1.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_1.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_1.q (working copy)
@@ -15,9 +15,11 @@
load data local inpath '../data/files/srcsortbucket4outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
+set hive.ignore.mapjoin.hint=true;
set hive.optimize.bucketmapjoin.sortedmerge = true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/mapjoin_hook.q
===================================================================
--- ql/src/test/queries/clientpositive/mapjoin_hook.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/mapjoin_hook.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
set hive.exec.post.hooks = org.apache.hadoop.hive.ql.hooks.MapJoinCounterHook ;
drop table dest1;
CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
Index: ql/src/test/queries/clientpositive/bucketmapjoin13.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin13.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin13.q (working copy)
@@ -22,6 +22,7 @@
INSERT OVERWRITE TABLE srcbucket_mapjoin_part_2 PARTITION (part='1')
SELECT * FROM src;
+set hive.ignore.mapjoin.hint=false;
set hive.optimize.bucketmapjoin=true;
-- part=1 partition for srcbucket_mapjoin_part_1 is bucketed by 'value'
Index: ql/src/test/queries/clientpositive/bucketmapjoin6.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin6.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin6.q (working copy)
@@ -3,12 +3,10 @@
create table tmp2 (a string, b string) clustered by (a) sorted by (a) into 10 buckets;
-
set hive.enforce.bucketing = true;
set hive.enforce.sorting = true;
set hive.exec.reducers.max=1;
-
insert overwrite table tmp1 select * from src where key < 50;
insert overwrite table tmp2 select * from src where key < 50;
Index: ql/src/test/queries/clientpositive/bucketcontext_5.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_5.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_5.q (working copy)
@@ -10,9 +10,11 @@
load data local inpath '../data/files/srcsortbucket2outof4.txt' INTO TABLE bucket_big;
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/smb_mapjoin_14_2.q
===================================================================
--- ql/src/test/queries/clientpositive/smb_mapjoin_14_2.q (revision 0)
+++ ql/src/test/queries/clientpositive/smb_mapjoin_14_2.q (working copy)
@@ -0,0 +1,245 @@
+set hive.enforce.bucketing = true;
+set hive.enforce.sorting = true;
+set hive.exec.reducers.max = 1;
+set hive.ignore.mapjoin.hint = true;
+
+CREATE TABLE tbl1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS;
+CREATE TABLE tbl2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS;
+
+insert overwrite table tbl1
+select * from src where key < 10;
+
+insert overwrite table tbl2
+select * from src where key < 10;
+
+set hive.optimize.bucketmapjoin = true;
+set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+
+-- All the mapjoin hints would be ignored
+-- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1;
+
+select count(*) from (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1;
+
+-- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
+-- Add a order by at the end to make the results deterministic.
+explain
+select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key;
+
+select key, count(*) from
+(
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+) subq1
+group by key
+order by key;
+
+-- The mapjoin is being performed as part of more than one sub-query. It should be converted to a sort-merge join
+explain
+select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2;
+
+select count(*) from
+(
+ select key, count(*) from
+ (
+ select /*+mapjoin(a)*/ a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key
+ ) subq1
+ group by key
+) subq2;
+
+-- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key;
+
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key;
+
+-- The subquery itself is being map-joined. Since the sub-query only contains selects and filters, it should
+-- be converted to a sort-merge join, although there is more than one level of sub-query
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key;
+
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join tbl2 b
+ on subq2.key = b.key;
+
+-- Both the big table and the small table are nested sub-queries i.e more then 1 level of sub-query.
+-- The join should be converted to a sort-merge join
+explain
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key;
+
+select /*+mapjoin(subq2)*/ count(*) from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+ join
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq3
+ where key < 6
+ ) subq4
+ on subq2.key = subq4.key;
+
+-- The subquery itself is being map-joined. Since the sub-query only contains selects and filters and the join key
+-- is not getting modified, it should be converted to a sort-merge join. Note that the sub-query modifies one
+-- item, but that is not part of the join key.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key;
+
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, concat(a.value, a.value) as value from tbl1 a where key < 8) subq1
+ join
+ (select a.key as key, concat(a.value, a.value) as value from tbl2 a where key < 8) subq2
+ on subq1.key = subq2.key;
+
+-- Since the join key is modified by the sub-query, neither sort-merge join not bucketized map-side
+-- join should be performed
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key;
+
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl1 a) subq1
+ join
+ (select a.key +1 as key, concat(a.value, a.value) as value from tbl2 a) subq2
+ on subq1.key = subq2.key;
+
+-- The small table is a sub-query and the big table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key;
+
+select /*+mapjoin(subq1)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key;
+
+-- The big table is a sub-query and the small table is not.
+-- It should be converted to a sort-merge join.
+explain
+select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key;
+
+select /*+mapjoin(a)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join tbl2 a on subq1.key = a.key;
+
+-- There are more than 2 inputs to the join, all of them being sub-queries.
+-- It should be converted to to a sort-merge join
+explain
+select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on (subq1.key = subq2.key)
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key);
+
+select /*+mapjoin(subq1, subq2)*/ count(*) from
+ (select a.key as key, a.value as value from tbl1 a where key < 6) subq1
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq2
+ on subq1.key = subq2.key
+ join
+ (select a.key as key, a.value as value from tbl2 a where key < 6) subq3
+ on (subq1.key = subq3.key);
+
+-- The mapjoin is being performed on a nested sub-query, and an aggregation is performed after that.
+-- The join should be converted to a sort-merge join
+explain
+select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a;
+
+select count(*) from (
+ select /*+mapjoin(subq2)*/ subq2.key as key, subq2.value as value1, b.value as value2 from
+ (
+ select * from
+ (
+ select a.key as key, a.value as value from tbl1 a where key < 8
+ ) subq1
+ where key < 6
+ ) subq2
+join tbl2 b
+on subq2.key = b.key) a;
Index: ql/src/test/queries/clientpositive/union22.q
===================================================================
--- ql/src/test/queries/clientpositive/union22.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/union22.q (working copy)
@@ -12,6 +12,7 @@
set hive.auto.convert.join=true;
set hive.auto.convert.join.noconditionaltask=true;
set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.ignore.mapjoin.hint=false;
-- Since the inputs are small, it should be automatically converted to mapjoin
Index: ql/src/test/queries/clientpositive/join40.q
===================================================================
--- ql/src/test/queries/clientpositive/join40.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join40.q (working copy)
@@ -30,6 +30,7 @@
SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value;
set hive.mapjoin.cache.numrows=2;
+set hive.ignore.mapjoin.hint=false;
EXPLAIN
SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value
Index: ql/src/test/queries/clientpositive/join26.q
===================================================================
--- ql/src/test/queries/clientpositive/join26.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join26.q (working copy)
@@ -1,4 +1,5 @@
CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE;
+set hive.ignore.mapjoin.hint=false;
EXPLAIN EXTENDED
INSERT OVERWRITE TABLE dest_j1
Index: ql/src/test/queries/clientpositive/mapjoin_distinct.q
===================================================================
--- ql/src/test/queries/clientpositive/mapjoin_distinct.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/mapjoin_distinct.q (working copy)
@@ -1,5 +1,7 @@
set hive.map.aggr = true;
set hive.groupby.skewindata = true;
+set hive.ignore.mapjoin.hint=false;
+
explain
FROM srcpart c
JOIN srcpart d
Index: ql/src/test/queries/clientpositive/join37.q
===================================================================
--- ql/src/test/queries/clientpositive/join37.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join37.q (working copy)
@@ -1,7 +1,6 @@
set hive.mapjoin.numrows = 2;
+set hive.ignore.mapjoin.hint=false;
-
-
CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE;
EXPLAIN
@@ -14,6 +13,3 @@
FROM src1 x JOIN src y ON (x.key = y.key);
select * from dest_j1 x order by x.key;
-
-
-
Index: ql/src/test/queries/clientpositive/bucketmapjoin10.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin10.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin10.q (working copy)
@@ -23,6 +23,7 @@
ALTER TABLE srcbucket_mapjoin_part_2 CLUSTERED BY (key) INTO 3 BUCKETS;
set hive.optimize.bucketmapjoin=true;
+set hive.ignore.mapjoin.hint=false;
-- The table bucketing metadata matches but the partition metadata does not, bucket map join should not be used
Index: ql/src/test/queries/clientpositive/smb_mapjoin_14.q
===================================================================
--- ql/src/test/queries/clientpositive/smb_mapjoin_14.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/smb_mapjoin_14.q (working copy)
@@ -14,6 +14,7 @@
set hive.optimize.bucketmapjoin = true;
set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set hive.ignore.mapjoin.hint=false;
-- The mapjoin is being performed as part of sub-query. It should be converted to a sort-merge join
explain
Index: ql/src/test/queries/clientpositive/bucketmapjoin3.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin3.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin3.q (working copy)
@@ -1,3 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
+
CREATE TABLE srcbucket_mapjoin(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin;
load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin;
Index: ql/src/test/queries/clientpositive/bucketmapjoin_negative2.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin_negative2.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin_negative2.q (working copy)
@@ -8,6 +8,7 @@
load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part_2 partition(ds='2008-04-09');
load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part_2 partition(ds='2008-04-09');
+set hive.ignore.mapjoin.hint=false;
set hive.optimize.bucketmapjoin = true;
create table bucketmapjoin_tmp_result (key string , value1 string, value2 string);
Index: ql/src/test/queries/clientpositive/semijoin.q
===================================================================
--- ql/src/test/queries/clientpositive/semijoin.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/semijoin.q (working copy)
@@ -1,8 +1,4 @@
-
-
-
-
-
+set hive.ignore.mapjoin.hint=false;
create table t1 as select cast(key as int) key, value from src where key <= 10;
select * from t1 sort by key;
@@ -76,8 +72,3 @@
explain select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key;
select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key;
-
-
-
-
-
Index: ql/src/test/queries/clientpositive/bucketcontext_2.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_2.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_2.q (working copy)
@@ -13,9 +13,11 @@
load data local inpath '../data/files/srcsortbucket2outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/bucketmapjoin7.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin7.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin7.q (working copy)
@@ -11,6 +11,7 @@
LOAD DATA LOCAL INPATH '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part_2 PARTITION (ds='2008-04-08', hr='0');
set hive.optimize.bucketmapjoin=true;
+set hive.ignore.mapjoin.hint=false;
-- Tests that bucket map join works with a table with more than one level of partitioning
Index: ql/src/test/queries/clientpositive/bucketcontext_6.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_6.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_6.q (working copy)
@@ -13,9 +13,11 @@
load data local inpath '../data/files/srcsortbucket2outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/join30.q
===================================================================
--- ql/src/test/queries/clientpositive/join30.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join30.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key INT, cnt INT);
EXPLAIN
Index: ql/src/test/queries/clientpositive/skewjoin.q
===================================================================
--- ql/src/test/queries/clientpositive/skewjoin.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/skewjoin.q (working copy)
@@ -1,12 +1,7 @@
set hive.optimize.skewjoin = true;
set hive.skewjoin.key = 2;
+set hive.ignore.mapjoin.hint=false;
-
-
-
-
-
-
CREATE TABLE T1(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE;
@@ -129,9 +124,3 @@
SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 JOIN T2 src2 ON src1.key+1 = src2.key JOIN T2 src3 ON src2.key = src3.key;
select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key;
-
-
-
-
-
-
Index: ql/src/test/queries/clientpositive/join27.q
===================================================================
--- ql/src/test/queries/clientpositive/join27.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join27.q (working copy)
@@ -1,5 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
-
CREATE TABLE dest_j1(key INT, value STRING, val2 STRING) STORED AS TEXTFILE;
EXPLAIN
Index: ql/src/test/queries/clientpositive/mapjoin1.q
===================================================================
--- ql/src/test/queries/clientpositive/mapjoin1.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/mapjoin1.q (working copy)
@@ -1,4 +1,5 @@
set hive.mapjoin.cache.numrows=100;
+set hive.ignore.mapjoin.hint=false;
SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
FROM srcpart a
Index: ql/src/test/queries/clientpositive/join38.q
===================================================================
--- ql/src/test/queries/clientpositive/join38.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join38.q (working copy)
@@ -1,5 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
-
create table tmp(col0 string, col1 string,col2 string,col3 string,col4 string,col5 string,col6 string,col7 string,col8 string,col9 string,col10 string,col11 string);
insert overwrite table tmp select key, cast(key + 1 as int), key +2, key+3, key+4, cast(key+5 as int), key+6, key+7, key+8, key+9, key+10, cast(key+11 as int) from src where key = 100;
@@ -16,5 +16,3 @@
SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count
where b.col11 = 111
group by a.value, b.col5;
-
-
Index: ql/src/test/queries/clientpositive/bucketmapjoin11.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin11.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin11.q (working copy)
@@ -22,7 +22,7 @@
LOAD DATA LOCAL INPATH '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part_2 PARTITION (part='2');
LOAD DATA LOCAL INPATH '../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part_2 PARTITION (part='2');
-
+set hive.ignore.mapjoin.hint=false;
set hive.optimize.bucketmapjoin=true;
-- The table and partition bucketing metadata doesn't match but the bucket numbers of all partitions is
Index: ql/src/test/queries/clientpositive/partition_wise_fileformat14.q
===================================================================
--- ql/src/test/queries/clientpositive/partition_wise_fileformat14.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/partition_wise_fileformat14.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
set hive.enforce.bucketing = true;
set hive.enforce.sorting = true;
set hive.exec.reducers.max = 1;
Index: ql/src/test/queries/clientpositive/bucketmapjoin4.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin4.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin4.q (working copy)
@@ -1,3 +1,5 @@
+set hive.ignore.mapjoin.hint=false;
+
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
CREATE TABLE srcbucket_mapjoin(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin;
Index: ql/src/test/queries/clientpositive/bucketmapjoin_negative3.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin_negative3.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin_negative3.q (working copy)
@@ -25,6 +25,7 @@
load data local inpath '../data/files/srcbucket22.txt' INTO TABLE test4;
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
-- should be allowed
explain extended select /* + MAPJOIN(R) */ * from test1 L join test1 R on L.key=R.key AND L.value=R.value;
explain extended select /* + MAPJOIN(R) */ * from test2 L join test2 R on L.key=R.key AND L.value=R.value;
Index: ql/src/test/queries/clientpositive/bucketcontext_3.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_3.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_3.q (working copy)
@@ -13,9 +13,11 @@
load data local inpath '../data/files/srcsortbucket4outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-08');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/bucketmapjoin8.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin8.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin8.q (working copy)
@@ -13,6 +13,7 @@
ALTER TABLE srcbucket_mapjoin_part_2 CLUSTERED BY (key) INTO 3 BUCKETS;
set hive.optimize.bucketmapjoin=true;
+set hive.ignore.mapjoin.hint=false;
-- The partition bucketing metadata match but the tables have different numbers of buckets, bucket map join should still be used
Index: ql/src/test/queries/clientpositive/bucketcontext_7.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_7.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_7.q (working copy)
@@ -18,9 +18,11 @@
load data local inpath '../data/files/srcsortbucket2outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientpositive/bucketizedhiveinputformat_auto.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketizedhiveinputformat_auto.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketizedhiveinputformat_auto.q (working copy)
@@ -14,9 +14,11 @@
load data local inpath '../data/files/srcsortbucket4outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-09');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.input.format = org.apache.hadoop.hive.ql.io.HiveInputFormat;
Index: ql/src/test/queries/clientpositive/join35.q
===================================================================
--- ql/src/test/queries/clientpositive/join35.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join35.q (working copy)
@@ -28,4 +28,25 @@
select * from dest_j1 x order by x.key;
+-- Mapjoin followed by union is not supported.
+-- The same query would work without the hint, or if the hint is ignored
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key);
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x) */ x.key, x.value, subq1.cnt
+FROM
+( SELECT x.key as key, count(1) as cnt from src x where x.key < 20 group by x.key
+ UNION ALL
+ SELECT x1.key as key, count(1) as cnt from src x1 where x1.key > 100 group by x1.key
+) subq1
+JOIN src1 x ON (x.key = subq1.key);
+
+select * from dest_j1 x order by x.key;
Index: ql/src/test/queries/clientpositive/join28.q
===================================================================
--- ql/src/test/queries/clientpositive/join28.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join28.q (working copy)
@@ -3,6 +3,7 @@
set hive.auto.convert.join=true;
set hive.auto.convert.join.noconditionaltask=true;
set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.ignore.mapjoin.hint=false;
-- Since the inputs are small, it should be automatically converted to mapjoin
@@ -23,5 +24,23 @@
select * from dest_j1 x order by x.key;
+set hive.ignore.mapjoin.hint=true;
+-- Mapjoin followed by mapjoin is not supported.
+-- The same query would work fine without the hint, or if the hint is ignored.
+EXPLAIN
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11);
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(z) */ subq.key1, z.value
+FROM
+(SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2
+ FROM src1 x JOIN src y ON (x.key = y.key)) subq
+ JOIN srcpart z ON (subq.key1 = z.key and z.ds='2008-04-08' and z.hr=11);
+
+select * from dest_j1 x order by x.key;
Index: ql/src/test/queries/clientpositive/bucketmapjoin1.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin1.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin1.q (working copy)
@@ -3,7 +3,7 @@
CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
-
+set hive.ignore.mapjoin.hint=false;
set hive.optimize.bucketmapjoin = true;
-- empty partitions (HIVE-3205)
Index: ql/src/test/queries/clientpositive/join39.q
===================================================================
--- ql/src/test/queries/clientpositive/join39.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/join39.q (working copy)
@@ -1,7 +1,6 @@
set hive.mapjoin.cache.numrows = 2;
+set hive.ignore.mapjoin.hint=false;
-
-
CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE;
explain
@@ -15,6 +14,3 @@
FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key);
select * from dest_j1 x order by x.key;
-
-
-
Index: ql/src/test/queries/clientpositive/bucketmapjoin12.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin12.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin12.q (working copy)
@@ -20,8 +20,9 @@
ALTER TABLE srcbucket_mapjoin_part_3 CLUSTERED BY (key) INTO 2 BUCKETS;
set hive.optimize.bucketmapjoin=true;
+set hive.ignore.mapjoin.hint=false;
--- The partition bucketing metadata match but one table is not bucketed, bucket map join should still be used
+-- Dont ignore the mapjoin hint. Look at HIVE-4042 for details.
EXPLAIN EXTENDED
SELECT /*+ MAPJOIN(b) */ count(*)
Index: ql/src/test/queries/clientpositive/stats11.q
===================================================================
--- ql/src/test/queries/clientpositive/stats11.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/stats11.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
set datanucleus.cache.collections=false;
set hive.stats.autogather=true;
Index: ql/src/test/queries/clientpositive/bucketmapjoin5.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketmapjoin5.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketmapjoin5.q (working copy)
@@ -22,6 +22,7 @@
create table bucketmapjoin_hash_result_2 (key bigint , value1 bigint, value2 bigint);
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
create table bucketmapjoin_tmp_result (key string , value1 string, value2 string);
explain extended
Index: ql/src/test/queries/clientpositive/bucketcontext_4.q
===================================================================
--- ql/src/test/queries/clientpositive/bucketcontext_4.q (revision 1459200)
+++ ql/src/test/queries/clientpositive/bucketcontext_4.q (working copy)
@@ -15,9 +15,11 @@
load data local inpath '../data/files/srcsortbucket2outof4.txt' INTO TABLE bucket_big partition(ds='2008-04-08');
set hive.optimize.bucketmapjoin = true;
+set hive.ignore.mapjoin.hint=false;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.ignore.mapjoin.hint=true;
explain extended select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
select /* + MAPJOIN(a) */ count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key;
Index: ql/src/test/queries/clientnegative/union22.q
===================================================================
--- ql/src/test/queries/clientnegative/union22.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/union22.q (working copy)
@@ -8,6 +8,7 @@
select key, key, value, key, value, value from src;
set hive.merge.mapfiles=false;
+set hive.ignore.mapjoin.hint=false;
-- Union followed by Mapjoin is not supported.
-- The same query would work without the hint
Index: ql/src/test/queries/clientnegative/invalid_mapjoin1.q
===================================================================
--- ql/src/test/queries/clientnegative/invalid_mapjoin1.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/invalid_mapjoin1.q (working copy)
@@ -1 +1,2 @@
+set hive.ignore.mapjoin.hint=false;
select /*+ MAPJOIN(a) ,MAPJOIN(b)*/ * from src a join src b on (a.key=b.key and a.value=b.value);
Index: ql/src/test/queries/clientnegative/join32.q
===================================================================
--- ql/src/test/queries/clientnegative/join32.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/join32.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE;
-- Mapjoin followed by Mapjoin is not supported.
Index: ql/src/test/queries/clientnegative/join35.q
===================================================================
--- ql/src/test/queries/clientnegative/join35.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/join35.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key STRING, value STRING, val2 INT) STORED AS TEXTFILE;
-- Mapjoin followed by union is not supported.
Index: ql/src/test/queries/clientnegative/join28.q
===================================================================
--- ql/src/test/queries/clientnegative/join28.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/join28.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key STRING, value STRING) STORED AS TEXTFILE;
-- Mapjoin followed by mapjoin is not supported.
Index: ql/src/test/queries/clientnegative/join29.q
===================================================================
--- ql/src/test/queries/clientnegative/join29.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/join29.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
CREATE TABLE dest_j1(key STRING, cnt1 INT, cnt2 INT);
-- Mapjoin followed by group by is not supported.
Index: ql/src/test/queries/clientnegative/join2.q
===================================================================
--- ql/src/test/queries/clientnegative/join2.q (revision 1459200)
+++ ql/src/test/queries/clientnegative/join2.q (working copy)
@@ -1,3 +1,4 @@
+set hive.ignore.mapjoin.hint=false;
SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value
FROM src1 x LEFT OUTER JOIN src y ON (x.key = y.key);
Index: ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (revision 1459200)
+++ ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (working copy)
@@ -43,6 +43,7 @@
boolean hasDistributeBy = false;
boolean hasClusterBy = false;
+ boolean mapJoinRemoved = false;
public boolean hasJoin() {
return hasJoin;
@@ -107,4 +108,12 @@
public void setHasClusterBy(boolean hasClusterBy) {
this.hasClusterBy = hasClusterBy;
}
+
+ public boolean isMapJoinRemoved() {
+ return mapJoinRemoved;
+ }
+
+ public void setMapJoinRemoved(boolean mapJoinRemoved) {
+ this.mapJoinRemoved = mapJoinRemoved;
+ }
}
Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1459200)
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy)
@@ -6096,19 +6096,27 @@
private List getMapSideJoinTables(QB qb) {
List cols = new ArrayList();
+
+
ASTNode hints = qb.getParseInfo().getHints();
for (int pos = 0; pos < hints.getChildCount(); pos++) {
ASTNode hint = (ASTNode) hints.getChild(pos);
if (((ASTNode) hint.getChild(0)).getToken().getType() == HiveParser.TOK_MAPJOIN) {
- ASTNode hintTblNames = (ASTNode) hint.getChild(1);
- int numCh = hintTblNames.getChildCount();
- for (int tblPos = 0; tblPos < numCh; tblPos++) {
- String tblName = ((ASTNode) hintTblNames.getChild(tblPos)).getText()
- .toLowerCase();
- if (!cols.contains(tblName)) {
- cols.add(tblName);
+ // the user has specified to ignore mapjoin hint
+ if (!conf.getBoolVar(HiveConf.ConfVars.HIVEIGNOREMAPJOINHINT)) {
+ ASTNode hintTblNames = (ASTNode) hint.getChild(1);
+ int numCh = hintTblNames.getChildCount();
+ for (int tblPos = 0; tblPos < numCh; tblPos++) {
+ String tblName = ((ASTNode) hintTblNames.getChild(tblPos)).getText()
+ .toLowerCase();
+ if (!cols.contains(tblName)) {
+ cols.add(tblName);
+ }
}
}
+ else {
+ queryProperties.setMapJoinRemoved(true);
+ }
}
}