Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1034599)
+++ conf/hive-default.xml (working copy)
@@ -473,7 +473,45 @@
When the average output file size of a job is less than this number, Hive will start an additional map-reduce job to merge the output files into bigger files. This is only done for map-only jobs if hive.merge.mapfiles is true, and for map-reduce jobs if hive.merge.mapredfiles is true.
+
+ hive.mapjoin.hashtable.initialCapacity
+ 100000
+ In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the initialCapacity of the in-memory hashtable
+
+
+
+ hive.mapjoin.hashtable.loadfactor
+ 0.75
+ In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the load factor for the in-memory hashtable
+
+
+
+ hive.mapjoin.smalltable.filesize
+ 25000000
+ The threshold for the input file size of the small tables; if the file size is smaller than this threshold, it will try to convert the common join into map join
+
+
+
+ hive.mapjoin.localtask.max.memory.usage
+ 0.90
+ This number means how much memory the local task can take to hold the key/value into in-memory hash table; If the local task's memory usage is more than this number, the local task will be abort by themself. It means the data of small table is too large to be hold in the memory.
+
+
+
+ hive.mapjoin.check.memory.rows
+ 100000
+ The number means after how many rows processed it needs to check the memory usage
+
+
+
+ hive.auto.convert.join
+ false
+ Whether Hive enable the optimization about converting common join into mapjoin based on the input file size
+
+
+
+
hive.script.auto.progress
false
Whether Hive Tranform/Map/Reduce Clause should automatically send progress information to TaskTracker to avoid the task getting killed because of inactivity. Hive sends progress information when the script is outputting to stderr. This option removes the need of periodically producing stderr messages, but users should be cautious because this may prevent infinite loops in the scripts to be killed by TaskTracker.
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1034599)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -231,6 +231,8 @@
// mapper/reducer memory in local mode
HIVEHADOOPMAXMEM("hive.mapred.local.mem", 0),
+ //small table file size
+ HIVESMALLTABLESFILESIZE("hive.smalltable.filesize",25000000L), //25M
// test mode in hive mode
HIVETESTMODE("hive.test.mode", false),
HIVETESTMODEPREFIX("hive.test.mode.prefix", "test_"),
@@ -243,6 +245,7 @@
HIVEMERGEMAPFILESAVGSIZE("hive.merge.smallfiles.avgsize", (long) (16 * 1000 * 1000)),
HIVESKEWJOIN("hive.optimize.skewjoin", false),
+ HIVECONVERTJOIN("hive.auto.convert.join", false),
HIVESKEWJOINKEY("hive.skewjoin.key", 1000000),
HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000),
HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432), //32M
@@ -251,7 +254,13 @@
HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000),
HIVEMAXMAPJOINSIZE("hive.mapjoin.maxsize", 100000),
+ HIVEHASHTABLETHRESHOLD("hive.hashtable.initialCapacity", 100000),
+ HIVEHASHTABLELOADFACTOR("hive.hashtable.loadfactor", (float) 0.75),
+ HIVEHASHTABLEMAXMEMORYUSAGE("hive.hashtable.max.memory.usage", (float) 0.90),
+ HIVEHASHTABLESCALE("hive.hashtable.scale", (long)100000),
+ HIVEDEBUGLOCALTASK("hive.debug.localtask",false),
+
HIVEJOBPROGRESS("hive.task.progress", false),
HIVEINPUTFORMAT("hive.input.format", ""),
Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (revision 1034599)
+++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (working copy)
@@ -187,6 +187,7 @@
return copyToStandardObject(o, oi, ObjectInspectorCopyOption.DEFAULT);
}
+
public static Object copyToStandardObject(Object o, ObjectInspector oi,
ObjectInspectorCopyOption objectInspectorOption) {
if (o == null) {
Index: ql/src/test/results/clientpositive/auto_join3.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join3.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join3.q.out (revision 0)
@@ -0,0 +1,403 @@
+PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key))) (TOK_TABREF src src3) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src3) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-11, Stage-1
+ Stage-9 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-9
+ Stage-0 depends on stages: Stage-1, Stage-5, Stage-6, Stage-7
+ Stage-2 depends on stages: Stage-0
+ Stage-10 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-10
+ Stage-11 has a backup stage: Stage-1
+ Stage-7 depends on stages: Stage-11
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 0
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col9
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-11
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src3
+ TableScan
+ alias: src3
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1
+ 2 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col9
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ src3
+ TableScan
+ alias: src3
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 2
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {VALUE._col0}
+ 1
+ 2 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col9
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-49-44_524_1112752922854683245/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-49-44_524_1112752922854683245/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+344360994461
Index: ql/src/test/results/clientpositive/auto_join13.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join13.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join13.q.out (revision 0)
@@ -0,0 +1,526 @@
+PREHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c2)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c4)))) src2) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c3)) (< (. (TOK_TABLE_OR_COL src1) c1) 100))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c5) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c6)))) src3) (AND (= (+ (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c3)) (. (TOK_TABLE_OR_COL src3) c5)) (< (. (TOK_TABLE_OR_COL src3) c5) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c4)))))))
+
+STAGE DEPENDENCIES:
+ Stage-11 is a root stage , consists of Stage-14, Stage-15, Stage-1
+ Stage-14 has a backup stage: Stage-1
+ Stage-9 depends on stages: Stage-14
+ Stage-8 depends on stages: Stage-1, Stage-9, Stage-10 , consists of Stage-12, Stage-13, Stage-2
+ Stage-12 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-12
+ Stage-3 depends on stages: Stage-2, Stage-6, Stage-7
+ Stage-13 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-13
+ Stage-2
+ Stage-15 has a backup stage: Stage-1
+ Stage-10 depends on stages: Stage-15
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-11
+ Conditional Operator
+
+ Stage: Stage-14
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-9
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col2, _col3
+ Position of Big Table: 0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-12
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src3:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 200)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col3} {_col0}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col2]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0]()]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col3} {_col0}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col2]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0]()]
+ outputColumnNames: _col1, _col2
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col1
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col1))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-45-14_270_4703590494419049936/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-13
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col3} {_col0}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col2]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0]()]
+ Position of Big Table: 1
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 200)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col3} {_col0}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col2]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0]()]
+ outputColumnNames: _col1, _col2
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col1
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col1))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: (_col0 + _col2)
+ type: double
+ sort order: +
+ Map-reduce partition columns:
+ expr: (_col0 + _col2)
+ type: double
+ tag: 0
+ value expressions:
+ expr: _col3
+ type: string
+ expr: _col0
+ type: string
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 200)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: UDFToDouble(_col0)
+ type: double
+ sort order: +
+ Map-reduce partition columns:
+ expr: UDFToDouble(_col0)
+ type: double
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col1} {VALUE._col2}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col1, _col2
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col1
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col1))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-15
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-10
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col2, _col3
+ Position of Big Table: 1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-15_078_6751775098674020635/-mr-10000
+POSTHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-15_078_6751775098674020635/-mr-10000
+-97670109576
Index: ql/src/test/results/clientpositive/auto_join_filters.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join_filters.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join_filters.q.out (revision 0)
@@ -0,0 +1,508 @@
+PREHOOK: query: CREATE TABLE myinput1(key int, value int)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE myinput1(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in3.txt' INTO TABLE myinput1
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in3.txt' INTO TABLE myinput1
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-54-58_117_4655002105750219/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-54-58_117_4655002105750219/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-02_366_6992699161975993053/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-02_366_6992699161975993053/-mr-10000
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-06_634_6445437890696840782/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-06_634_6445437890696840782/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-11_343_4548102731373306036/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-11_343_4548102731373306036/-mr-10000
+19749880
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-16_548_5347533565842262214/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-16_548_5347533565842262214/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-20_995_7002425294061259326/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-20_995_7002425294061259326/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-25_522_7223107408677755461/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-25_522_7223107408677755461/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-29_880_4549706398893829025/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-29_880_4549706398893829025/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-34_424_169089929568714471/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-34_424_169089929568714471/-mr-10000
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-38_712_4712108057879519702/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-38_712_4712108057879519702/-mr-10000
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-44_225_1720576234903554068/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-44_225_1720576234903554068/-mr-10000
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-49_018_1771521729720905137/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-49_018_1771521729720905137/-mr-10000
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-53_798_4683803195318455359/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-53_798_4683803195318455359/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-58_327_8236566722650852723/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-55-58_327_8236566722650852723/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-03_065_2996745104714988105/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-03_065_2996745104714988105/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-07_973_4625804318414132050/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-07_973_4625804318414132050/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-12_356_8298086109390383704/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-12_356_8298086109390383704/-mr-10000
+4939870
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-17_229_5664599811337231023/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-17_229_5664599811337231023/-mr-10000
+4939870
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-21_523_7285750866206869237/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-21_523_7285750866206869237/-mr-10000
+4939870
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-26_625_2655349310089115439/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-26_625_2655349310089115439/-mr-10000
+4939870
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-32_045_6195138915928486369/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-32_045_6195138915928486369/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-36_424_3499869040206387595/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-36_424_3499869040206387595/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-40_774_9079581832077075239/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-40_774_9079581832077075239/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-47_855_6954543840613097127/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-47_855_6954543840613097127/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-54_495_8347295964623851135/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-56-54_495_8347295964623851135/-mr-10000
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-02_103_2472217444100405726/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-02_103_2472217444100405726/-mr-10000
+3078400
+PREHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@smb_input1
+PREHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@smb_input2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input1
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input1
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@smb_input1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input1
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input1
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@smb_input1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input2
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input2
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@smb_input2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input2
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input2
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@smb_input2
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-14_989_3018079175573168618/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-14_989_3018079175573168618/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-20_236_1394320099184495685/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-20_236_1394320099184495685/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-24_621_5150978026396273270/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-24_621_5150978026396273270/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-28_867_297793229671774013/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-28_867_297793229671774013/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-33_384_4213543405474846039/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-33_384_4213543405474846039/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-39_211_3398396582126586587/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-39_211_3398396582126586587/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-43_619_5340976824563811251/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-43_619_5340976824563811251/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-48_456_7729606752684080265/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-48_456_7729606752684080265/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-53_321_4161401378864306240/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-53_321_4161401378864306240/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-58_185_6149588741729351812/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-57-58_185_6149588741729351812/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-03_957_1701501074902739227/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-03_957_1701501074902739227/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-08_779_8730807341859621811/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-08_779_8730807341859621811/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-14_365_6978393466194392157/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-14_365_6978393466194392157/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-19_195_6596525098045181032/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-19_195_6596525098045181032/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-23_944_767168653859149753/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-23_944_767168653859149753/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-28_382_1684228097795355116/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-28_382_1684228097795355116/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-33_122_626333363788741642/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-33_122_626333363788741642/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-37_662_749514132507506514/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-37_662_749514132507506514/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-43_069_1206846342850380139/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-43_069_1206846342850380139/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-47_842_61082551066774054/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-47_842_61082551066774054/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-52_552_2495832598945335366/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-52_552_2495832598945335366/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-57_562_5897053800333413408/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-58-57_562_5897053800333413408/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-02_213_6341773528378956115/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-02_213_6341773528378956115/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-10_007_6693537617189622123/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-10_007_6693537617189622123/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-18_342_340633536253451649/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-18_342_340633536253451649/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-26_006_4411101160964366136/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-26_006_4411101160964366136/-mr-10000
+3078400
Index: ql/src/test/results/clientpositive/auto_join22.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join22.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join22.q.out (revision 0)
@@ -0,0 +1,420 @@
+PREHOOK: query: explain
+SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src4) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) src1_key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) src1_value)))) src3) (= (. (TOK_TABLE_OR_COL src3) src1_key) (. (TOK_TABLE_OR_COL src4) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src4) value) src4_value) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src4) key) src4_key)))) src5)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src5) src1_value)))))))
+
+STAGE DEPENDENCIES:
+ Stage-11 is a root stage , consists of Stage-14, Stage-15, Stage-1
+ Stage-14 has a backup stage: Stage-1
+ Stage-9 depends on stages: Stage-14
+ Stage-8 depends on stages: Stage-1, Stage-9, Stage-10 , consists of Stage-12, Stage-13, Stage-2
+ Stage-12 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-12
+ Stage-3 depends on stages: Stage-2, Stage-6, Stage-7
+ Stage-13 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-13
+ Stage-2
+ Stage-15 has a backup stage: Stage-1
+ Stage-10 depends on stages: Stage-15
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-11
+ Conditional Operator
+
+ Stage: Stage-14
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src5:src3:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src5:src3:src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-9
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src5:src3:src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-12
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {_col3}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col2]]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src5:src4
+ TableScan
+ alias: src4
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {_col3}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col2]]
+ outputColumnNames: _col7
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col7
+ type: string
+ outputColumnNames: _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: string
+ outputColumnNames: _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-48-06_098_1713922165829834361/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-13
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src5:src4
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src5:src4
+ TableScan
+ alias: src4
+ HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {_col3}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col2]]
+ Position of Big Table: 1
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {_col3}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col2]]
+ outputColumnNames: _col7
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col7
+ type: string
+ outputColumnNames: _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: string
+ outputColumnNames: _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col2
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col2
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col3
+ type: string
+ src5:src4
+ TableScan
+ alias: src4
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {VALUE._col3}
+ handleSkewJoin: false
+ outputColumnNames: _col7
+ Select Operator
+ expressions:
+ expr: _col7
+ type: string
+ outputColumnNames: _col3
+ Select Operator
+ expressions:
+ expr: _col3
+ type: string
+ outputColumnNames: _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-15
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src5:src3:src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src5:src3:src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-10
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src5:src3:src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src5:src3:src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src5:src3:src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-06_660_6147727634332204517/-mr-10000
+POSTHOOK: query: SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-06_660_6147727634332204517/-mr-10000
+344337359100
Index: ql/src/test/results/clientpositive/auto_join9.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join9.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join9.q.out (revision 0)
@@ -0,0 +1,276 @@
+PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12'
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF srcpart src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL src1) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL src1) hr) '12')))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-2 depends on stages: Stage-0
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Position of Big Table: 0
+ Filter Operator
+ predicate:
+ expr: ((_col2 = '2008-04-08') and (_col3 = '12'))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Position of Big Table: 1
+ Filter Operator
+ predicate:
+ expr: ((_col2 = '2008-04-08') and (_col3 = '12'))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: ds
+ type: string
+ expr: hr
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col2} {VALUE._col3}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Filter Operator
+ predicate:
+ expr: ((_col2 = '2008-04-08') and (_col3 = '12'))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-49_730_4731952848139618200/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-49_730_4731952848139618200/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+101861029915
Index: ql/src/test/results/clientpositive/auto_join19.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join19.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join19.q.out (revision 0)
@@ -0,0 +1,286 @@
+PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF srcpart src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value))) (TOK_WHERE (and (or (= (. (TOK_TABLE_OR_COL src1) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL src1) ds) '2008-04-09')) (or (= (. (TOK_TABLE_OR_COL src1) hr) '12') (= (. (TOK_TABLE_OR_COL src1) hr) '11'))))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-2 depends on stages: Stage-0
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Position of Big Table: 0
+ Filter Operator
+ predicate:
+ expr: (((_col2 = '2008-04-08') or (_col2 = '2008-04-09')) and ((_col3 = '12') or (_col3 = '11')))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {ds} {hr}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Position of Big Table: 1
+ Filter Operator
+ predicate:
+ expr: (((_col2 = '2008-04-08') or (_col2 = '2008-04-09')) and ((_col3 = '12') or (_col3 = '11')))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: ds
+ type: string
+ expr: hr
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col2} {VALUE._col3}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col2, _col3, _col7
+ Filter Operator
+ predicate:
+ expr: (((_col2 = '2008-04-08') or (_col2 = '2008-04-09')) and ((_col3 = '12') or (_col3 = '11')))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col7
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-03_485_5529985135227819485/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-03_485_5529985135227819485/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+407444119660
Index: ql/src/test/results/clientpositive/auto_join4.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join4.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join4.q.out (revision 0)
@@ -0,0 +1,343 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) c2)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src1) key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) c4)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src2) key) 15) (< (. (TOK_TABLE_OR_COL src2) key) 25))))) b) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL b) c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c1) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c2) c2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c3) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c4)))))
+
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage , consists of Stage-6, Stage-1
+ Stage-6 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-6
+ Stage-0 depends on stages: Stage-1, Stage-4
+ Stage-2 depends on stages: Stage-0
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-5
+ Conditional Operator
+
+ Stage: Stage-6
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ c:b:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-49-57_379_6521114373353088936/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-49-57_379_6521114373353088936/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+5079148035
Index: ql/src/test/results/clientpositive/auto_join14.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join14.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join14.q.out (revision 0)
@@ -0,0 +1,298 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src) (TOK_TABREF srcpart) (and (AND (= (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL srcpart) key)) (= (. (TOK_TABLE_OR_COL srcpart) ds) '2008-04-08')) (> (. (TOK_TABLE_OR_COL src) key) 100)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL srcpart) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-2 depends on stages: Stage-0
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ srcpart
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ srcpart
+ TableScan
+ alias: srcpart
+ Filter Operator
+ predicate:
+ expr: (ds = '2008-04-08')
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ srcpart
+ TableScan
+ alias: srcpart
+ Filter Operator
+ predicate:
+ expr: (ds = '2008-04-08')
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 100)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ srcpart
+ TableScan
+ alias: srcpart
+ Filter Operator
+ predicate:
+ expr: (ds = '2008-04-08')
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/data/users/liyintang/hive-1642/build/ql/scratchdir/hive_2010-11-15_16-45-41_871_1477229060958933224/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/data/users/liyintang/hive-1642/build/ql/scratchdir/hive_2010-11-15_16-45-41_871_1477229060958933224/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+404554174174
Index: ql/src/test/results/clientpositive/auto_join23.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join23.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join23.q.out (revision 0)
@@ -0,0 +1,374 @@
+PREHOOK: query: explain
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (< (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src2) key) 10))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src1) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src1) value)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src2) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src2) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 0
+ Filter Operator
+ predicate:
+ expr: ((_col0 < 10) and (_col4 < 10))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-48-22_254_5597843343584852831/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ sort order: ++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Filter Operator
+ predicate:
+ expr: ((_col0 < 10) and (_col4 < 10))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Filter Operator
+ predicate:
+ expr: ((_col0 < 10) and (_col4 < 10))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-22_537_89367652801535607/-mr-10000
+POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-22_537_89367652801535607/-mr-10000
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 0 val_0
+0 val_0 2 val_2
+0 val_0 2 val_2
+0 val_0 2 val_2
+0 val_0 4 val_4
+0 val_0 4 val_4
+0 val_0 4 val_4
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 8 val_8
+0 val_0 8 val_8
+0 val_0 8 val_8
+0 val_0 9 val_9
+0 val_0 9 val_9
+0 val_0 9 val_9
+2 val_2 0 val_0
+2 val_2 0 val_0
+2 val_2 0 val_0
+2 val_2 2 val_2
+2 val_2 4 val_4
+2 val_2 5 val_5
+2 val_2 5 val_5
+2 val_2 5 val_5
+2 val_2 8 val_8
+2 val_2 9 val_9
+4 val_4 0 val_0
+4 val_4 0 val_0
+4 val_4 0 val_0
+4 val_4 2 val_2
+4 val_4 4 val_4
+4 val_4 5 val_5
+4 val_4 5 val_5
+4 val_4 5 val_5
+4 val_4 8 val_8
+4 val_4 9 val_9
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 2 val_2
+5 val_5 2 val_2
+5 val_5 2 val_2
+5 val_5 4 val_4
+5 val_5 4 val_4
+5 val_5 4 val_4
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 8 val_8
+5 val_5 8 val_8
+5 val_5 8 val_8
+5 val_5 9 val_9
+5 val_5 9 val_9
+5 val_5 9 val_9
+8 val_8 0 val_0
+8 val_8 0 val_0
+8 val_8 0 val_0
+8 val_8 2 val_2
+8 val_8 4 val_4
+8 val_8 5 val_5
+8 val_8 5 val_5
+8 val_8 5 val_5
+8 val_8 8 val_8
+8 val_8 9 val_9
+9 val_9 0 val_0
+9 val_9 0 val_0
+9 val_9 0 val_0
+9 val_9 2 val_2
+9 val_9 4 val_4
+9 val_9 5 val_5
+9 val_9 5 val_5
+9 val_9 5 val_5
+9 val_9 8 val_8
+9 val_9 9 val_9
Index: ql/src/test/results/clientpositive/auto_join5.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join5.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join5.q.out (revision 0)
@@ -0,0 +1,343 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) c2)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src1) key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) c4)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src2) key) 15) (< (. (TOK_TABLE_OR_COL src2) key) 25))))) b) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL b) c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c1) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c2) c2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c3) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c4)))))
+
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage , consists of Stage-6, Stage-1
+ Stage-6 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-6
+ Stage-0 depends on stages: Stage-1, Stage-4
+ Stage-2 depends on stages: Stage-0
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-5
+ Conditional Operator
+
+ Stage: Stage-6
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ c:a:src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-10_093_3777419778879975109/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-10_093_3777419778879975109/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+9766083196
Index: ql/src/test/results/clientpositive/auto_join15.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join15.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join15.q.out (revision 0)
@@ -0,0 +1,318 @@
+PREHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2)))))))
+
+STAGE DEPENDENCIES:
+ Stage-7 is a root stage , consists of Stage-8, Stage-9, Stage-1
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
+ Stage-3 depends on stages: Stage-2
+ Stage-9 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-9
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-45-46_433_1629023500092721737/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ sort order: ++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-45-46_433_1629023500092721737/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ a:src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-46_758_4504467463212431923/-mr-10000
+POSTHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-46_758_4504467463212431923/-mr-10000
+524272996896
Index: ql/src/test/results/clientpositive/auto_join_nulls.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join_nulls.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join_nulls.q.out (revision 0)
@@ -0,0 +1,208 @@
+PREHOOK: query: CREATE TABLE myinput1(key int, value int)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE myinput1(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' INTO TABLE myinput1
+PREHOOK: type: LOAD
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/in1.txt' INTO TABLE myinput1
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-38_128_5721983744421015511/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-38_128_5721983744421015511/-mr-10000
+13630578
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-42_458_3857336078300862994/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-42_458_3857336078300862994/-mr-10000
+13630578
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-46_795_3295843614724111648/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-46_795_3295843614724111648/-mr-10000
+13630578
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-51_810_8495116173420785144/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-51_810_8495116173420785144/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-56_866_676999079349671986/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_17-59-56_866_676999079349671986/-mr-10000
+4509856
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-01_985_6481162350367357952/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-01_985_6481162350367357952/-mr-10000
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-07_433_5697822947001003360/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-07_433_5697822947001003360/-mr-10000
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-11_860_7663380874040287741/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-11_860_7663380874040287741/-mr-10000
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-17_087_5570220930038430711/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-17_087_5570220930038430711/-mr-10000
+4542038
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-21_730_7068579215030025497/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-21_730_7068579215030025497/-mr-10000
+4543491
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-26_763_7077135887434220096/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-26_763_7077135887434220096/-mr-10000
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-31_723_2846348272261226572/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-31_723_2846348272261226572/-mr-10000
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-37_879_1948558748358099991/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-37_879_1948558748358099991/-mr-10000
+4509891
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-42_711_8448804775949567435/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-42_711_8448804775949567435/-mr-10000
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-48_016_2711326187485329520/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-48_016_2711326187485329520/-mr-10000
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-52_763_9087751756999134388/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-52_763_9087751756999134388/-mr-10000
+4543526
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-58_006_5983144248451158873/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-00-58_006_5983144248451158873/-mr-10000
+4543526
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-03_223_3881722054830137933/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-03_223_3881722054830137933/-mr-10000
+4543526
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-07_938_7263469664808317293/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-07_938_7263469664808317293/-mr-10000
+4543526
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-12_851_3735155037354765439/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-12_851_3735155037354765439/-mr-10000
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-17_734_3064013262164657326/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-17_734_3064013262164657326/-mr-10000
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-22_200_5772028834197460678/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-11_18-01-22_200_5772028834197460678/-mr-10000
+3112070
Index: ql/src/test/results/clientpositive/auto_join24.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join24.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join24.q.out (revision 0)
@@ -0,0 +1,260 @@
+PREHOOK: query: create table tst1(key STRING, cnt INT)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tst1(key STRING, cnt INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tst1
+PREHOOK: query: INSERT OVERWRITE TABLE tst1
+SELECT a.key, count(1) FROM src a group by a.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tst1
+POSTHOOK: query: INSERT OVERWRITE TABLE tst1
+SELECT a.key, count(1) FROM src a group by a.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tst1
+POSTHOOK: Lineage: tst1.cnt EXPRESSION [(src)a.null, ]
+POSTHOOK: Lineage: tst1.key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: explain
+SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: tst1.cnt EXPRESSION [(src)a.null, ]
+POSTHOOK: Lineage: tst1.key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF tst1 a) (TOK_TABREF tst1 b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (. (TOK_TABLE_OR_COL a) cnt))))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ b
+ TableScan
+ alias: b
+ HashTable Sink Operator
+ condition expressions:
+ 0 {cnt}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {cnt}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col1
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col1
+ type: int
+ outputColumnNames: _col1
+ Group By Operator
+ aggregations:
+ expr: sum(_col1)
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-48-39_265_1289386780470095965/-mr-10002
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ HashTable Sink Operator
+ condition expressions:
+ 0 {cnt}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b
+ TableScan
+ alias: b
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {cnt}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col1
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col1
+ type: int
+ outputColumnNames: _col1
+ Group By Operator
+ aggregations:
+ expr: sum(_col1)
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: cnt
+ type: int
+ b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col1}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col1
+ Select Operator
+ expressions:
+ expr: _col1
+ type: int
+ outputColumnNames: _col1
+ Group By Operator
+ aggregations:
+ expr: sum(_col1)
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tst1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-39_524_8876213971940214888/-mr-10000
+POSTHOOK: query: SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tst1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-48-39_524_8876213971940214888/-mr-10000
+POSTHOOK: Lineage: tst1.cnt EXPRESSION [(src)a.null, ]
+POSTHOOK: Lineage: tst1.key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ]
+500
Index: ql/src/test/results/clientpositive/auto_join0.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join0.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join0.q.out (revision 0)
@@ -0,0 +1,408 @@
+PREHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 10)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (< (. (TOK_TABLE_OR_COL src) key) 10)))) src2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2)))))))
+
+STAGE DEPENDENCIES:
+ Stage-7 is a root stage , consists of Stage-8, Stage-9, Stage-1
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
+ Stage-3 depends on stages: Stage-2
+ Stage-9 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-9
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src2:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src2:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-44-13_985_3284286571612503277/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ sort order: ++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-44-13_985_3284286571612503277/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src2:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ a:src2:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-15_495_4888755931558413561/-mr-10000
+POSTHOOK: query: select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-15_495_4888755931558413561/-mr-10000
+34441656720
Index: ql/src/test/results/clientpositive/auto_join10.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join10.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join10.q.out (revision 0)
@@ -0,0 +1,303 @@
+PREHOOK: query: explain
+FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL Y) key) (. (TOK_TABLE_OR_COL Y) value)))))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ y:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ y:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ x:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-44-42_271_2728948276756488359/-mr-10002
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ x:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ x:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ y:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col2, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ x:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ y:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-42_676_7610566265023996906/-mr-10000
+POSTHOOK: query: FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-42_676_7610566265023996906/-mr-10000
+103231310608
Index: ql/src/test/results/clientpositive/auto_join6.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join6.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join6.q.out (revision 0)
@@ -0,0 +1,225 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) c2)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src1) key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) c4)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src2) key) 15) (< (. (TOK_TABLE_OR_COL src2) key) 25))))) b) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL b) c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c1) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c2) c2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c3) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c4)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Outer Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+
+PREHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-53-45_805_5842090138567718146/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-53-45_805_5842090138567718146/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+2607643291
Index: ql/src/test/results/clientpositive/auto_join16.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join16.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join16.q.out (revision 0)
@@ -0,0 +1,357 @@
+PREHOOK: query: explain
+SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) value))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL a) key) 10)))) subq) (TOK_TABREF src tab) (and (and (= (. (TOK_TABLE_OR_COL subq) key) (. (TOK_TABLE_OR_COL tab) key)) (> (. (TOK_TABLE_OR_COL subq) key) 20)) (= (. (TOK_TABLE_OR_COL subq) value) (. (TOK_TABLE_OR_COL tab) value))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL subq) key) (. (TOK_TABLE_OR_COL tab) value))))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL tab) value) 200))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ tab
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ tab
+ TableScan
+ alias: tab
+ Filter Operator
+ predicate:
+ expr: (value < 200)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0], Column[_col1]]
+ 1 [Column[key], Column[value]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key > 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate:
+ expr: (_col0 > 20)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0], Column[_col1]]
+ 1 [Column[key], Column[value]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 0
+ Filter Operator
+ predicate:
+ expr: (_col3 < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-45-59_829_630621006367329763/-mr-10002
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ subq:a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ subq:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key > 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate:
+ expr: (_col0 > 20)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0], Column[_col1]]
+ 1 [Column[key], Column[value]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tab
+ TableScan
+ alias: tab
+ Filter Operator
+ predicate:
+ expr: (value < 200)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0], Column[_col1]]
+ 1 [Column[key], Column[value]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 1
+ Filter Operator
+ predicate:
+ expr: (_col3 < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ subq:a
+ TableScan
+ alias: a
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key > 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate:
+ expr: (_col0 > 20)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ tab
+ TableScan
+ alias: tab
+ Filter Operator
+ predicate:
+ expr: (value < 200)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col3
+ Filter Operator
+ predicate:
+ expr: (_col3 < 200)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-00_194_7551947713427199304/-mr-10000
+POSTHOOK: query: SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-00_194_7551947713427199304/-mr-10000
+NULL
Index: ql/src/test/results/clientpositive/auto_join25.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join25.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join25.q.out (revision 0)
@@ -0,0 +1,112 @@
+PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-09-55_396_5931103673819276275/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-09-55_396_5931103673819276275/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+407444119660
+PREHOOK: query: CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_j2
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest_j2
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest_j2
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j2
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-10-05_166_1509512648391049274/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j2
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-10-05_166_1509512648391049274/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+33815990627
+PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-10-12_389_3216138172725864168/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-12_13-10-12_389_3216138172725864168/-mr-10000
+POSTHOOK: Lineage: dest1.key EXPRESSION [(srcpart)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+101861029915
Index: ql/src/test/results/clientpositive/auto_join1.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join1.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join1.q.out (revision 0)
@@ -0,0 +1,258 @@
+PREHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_j1
+PREHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest_j1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-2 depends on stages: Stage-0
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j1
+
+
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest_j1
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-38_075_6667574704570584214/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-38_075_6667574704570584214/-mr-10000
+POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+101861029915
Index: ql/src/test/results/clientpositive/auto_join11.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join11.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join11.q.out (revision 0)
@@ -0,0 +1,328 @@
+PREHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c2)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c4)))) src2) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c3)) (< (. (TOK_TABLE_OR_COL src1) c1) 100)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c4)))))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-44-52_532_6544376348022905932/-mr-10002
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-53_228_5586165183645136688/-mr-10000
+POSTHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-44-53_228_5586165183645136688/-mr-10000
+-101333194320
Index: ql/src/test/results/clientpositive/auto_join20.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join20.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join20.q.out (revision 0)
@@ -0,0 +1,938 @@
+PREHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (< (. (TOK_TABLE_OR_COL src1) key) 10))) (TOK_TABREF src src3) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src3) key)) (< (. (TOK_TABLE_OR_COL src3) key) 20)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) key) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value) v3)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k3)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v3))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2) (. (TOK_TABLE_OR_COL a) k3) (. (TOK_TABLE_OR_COL a) v3)))))))
+
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-1
+ Stage-9 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-9
+ Stage-2 depends on stages: Stage-1, Stage-6, Stage-7
+ Stage-3 depends on stages: Stage-2
+ Stage-10 has a backup stage: Stage-1
+ Stage-7 depends on stages: Stage-10
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1
+ Fetch Operator
+ limit: -1
+ a:src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+ a:src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-47-27_151_4779551438490700024/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ sort order: ++++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3,_col4,_col5))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-47-27_151_4779551438490700024/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1
+ Fetch Operator
+ limit: -1
+ a:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+ a:src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src3
+ TableScan
+ alias: src3
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ a:src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ a:src3
+ TableScan
+ alias: src3
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 2
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ 2 {VALUE._col0} {VALUE._col1}
+ filter predicates:
+ 0
+ 1
+ 2 {(VALUE._col0 < 20)}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-27_564_5189821371378694953/-mr-10000
+POSTHOOK: query: select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-27_564_5189821371378694953/-mr-10000
+56157587016
+PREHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (AND (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (< (. (TOK_TABLE_OR_COL src1) key) 10)) (< (. (TOK_TABLE_OR_COL src2) key) 15))) (TOK_TABREF src src3) (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src3) key)) (< (. (TOK_TABLE_OR_COL src3) key) 20)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) v1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) v2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) key) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value) v3)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL k3)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL v3))))) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL a) v1) (. (TOK_TABLE_OR_COL a) k2) (. (TOK_TABLE_OR_COL a) v2) (. (TOK_TABLE_OR_COL a) k3) (. (TOK_TABLE_OR_COL a) v3)))))))
+
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-1
+ Stage-9 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-9
+ Stage-2 depends on stages: Stage-1, Stage-6, Stage-7
+ Stage-3 depends on stages: Stage-2
+ Stage-10 has a backup stage: Stage-1
+ Stage-7 depends on stages: Stage-10
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1
+ Fetch Operator
+ limit: -1
+ a:src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+ a:src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-47-40_425_8206806234732005559/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ sort order: ++++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ Reduce Operator Tree:
+ Extract
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3,_col4,_col5))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-47-40_425_8206806234732005559/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a:src1
+ Fetch Operator
+ limit: -1
+ a:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+ a:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src3
+ TableScan
+ alias: src3
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1
+ 2 {(key < 20)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 10)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ a:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key < 15)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ a:src3
+ TableScan
+ alias: src3
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 2
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Right Outer Join0 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ 2 {VALUE._col0} {VALUE._col1}
+ filter predicates:
+ 0
+ 1
+ 2 {(VALUE._col0 < 20)}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-40_847_3956511255954544487/-mr-10000
+POSTHOOK: query: select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-40_847_3956511255954544487/-mr-10000
+56157587016
Index: ql/src/test/results/clientpositive/auto_join7.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join7.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join7.q.out (revision 0)
@@ -0,0 +1,295 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) c2)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src1) key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) c4)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src2) key) 15) (< (. (TOK_TABLE_OR_COL src2) key) 25))))) b) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL b) c3))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) key) c5) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value) c6)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src3) key) 20) (< (. (TOK_TABLE_OR_COL src3) key) 25))))) c) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL c) c5)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c1) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c2) c2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c3) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c4) c4) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c5) c5) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c6) c6)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c4)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c5)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c6)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:c:src3
+ TableScan
+ alias: src3
+ Filter Operator
+ predicate:
+ expr: ((key > 20) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 20) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 2
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Outer Join 0 to 1
+ Left Outer Join0 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ 2 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ expr: UDFToInteger(_col4)
+ type: int
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+
+PREHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c5 EXPRESSION [(src)src3.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c6 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4,dest1.c5,dest1.c6)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-22_059_1050419042564302638/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4,dest1.c5,dest1.c6)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-22_059_1050419042564302638/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c5 EXPRESSION [(src)src3.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c6 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+-5178357269
Index: ql/src/test/results/clientpositive/auto_join17.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join17.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join17.q.out (revision 0)
@@ -0,0 +1,290 @@
+PREHOOK: query: CREATE TABLE dest1(key1 INT, value1 STRING, key2 INT, value2 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key1 INT, value1 STRING, key2 INT, value2 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src1)) (TOK_SELEXPR (TOK_ALLCOLREF src2)))))
+
+STAGE DEPENDENCIES:
+ Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1
+ Stage-7 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-1, Stage-4, Stage-5
+ Stage-2 depends on stages: Stage-0
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.key1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value1 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.key1,dest1.value1,dest1.key2,dest1.value2)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-19_573_2083505126618399043/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.key1,dest1.value1,dest1.key2,dest1.value2)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-19_573_2083505126618399043/-mr-10000
+POSTHOOK: Lineage: dest1.key1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value1 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+-793937029770
Index: ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out (revision 0)
@@ -0,0 +1,302 @@
+PREHOOK: query: explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) key) (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src1) value)) value)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src1) key)))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src1 src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) key) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL src2) value)) value1) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL src2) key)) value2)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src2) key)))) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL b) key) (. (TOK_TABLE_OR_COL b) value1) (. (TOK_TABLE_OR_COL b) value2)))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-3 depends on stages: Stage-2
+ Stage-4 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b:src2
+ TableScan
+ alias: src2
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT value)
+ expr: count(DISTINCT key)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col2
+ type: bigint
+ expr: _col3
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT KEY._col1:0._col0)
+ expr: count(DISTINCT KEY._col1:1._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col1:1._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: bigint
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: bigint
+ $INTNAME1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Outer Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ expr: _col4
+ type: bigint
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3,_col4))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-46-38_518_4441580357260614840/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-38_684_1519279081596807783/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-38_684_1519279081596807783/-mr-10000
+14748607855
Index: ql/src/test/results/clientpositive/auto_join2.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join2.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join2.q.out (revision 0)
@@ -0,0 +1,404 @@
+PREHOOK: query: CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest_j2
+PREHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key))) (TOK_TABREF src src3) (= (+ (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (. (TOK_TABLE_OR_COL src3) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest_j2)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src3) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-11 is a root stage , consists of Stage-14, Stage-15, Stage-1
+ Stage-14 has a backup stage: Stage-1
+ Stage-9 depends on stages: Stage-14
+ Stage-8 depends on stages: Stage-1, Stage-9, Stage-10 , consists of Stage-12, Stage-13, Stage-2
+ Stage-12 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-12
+ Stage-0 depends on stages: Stage-2, Stage-6, Stage-7
+ Stage-3 depends on stages: Stage-0
+ Stage-13 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-13
+ Stage-2
+ Stage-15 has a backup stage: Stage-1
+ Stage-10 depends on stages: Stage-15
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-11
+ Conditional Operator
+
+ Stage: Stage-14
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {key}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-9
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {key}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col4
+ Position of Big Table: 0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-12
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col4]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ Position of Big Table: 0
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col4]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ outputColumnNames: _col4, _col9
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j2
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j2
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+ Stage: Stage-13
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col4]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ Position of Big Table: 1
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src3
+ TableScan
+ alias: src3
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0}
+ 1 {value}
+ handleSkewJoin: false
+ keys:
+ 0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[_col0], Column[_col4]()]
+ 1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+ outputColumnNames: _col4, _col9
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j2
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: (_col0 + _col4)
+ type: double
+ sort order: +
+ Map-reduce partition columns:
+ expr: (_col0 + _col4)
+ type: double
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ src3
+ TableScan
+ alias: src3
+ Reduce Output Operator
+ key expressions:
+ expr: UDFToDouble(key)
+ type: double
+ sort order: +
+ Map-reduce partition columns:
+ expr: UDFToDouble(key)
+ type: double
+ tag: 1
+ value expressions:
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col4}
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col4, _col9
+ Select Operator
+ expressions:
+ expr: _col4
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest_j2
+
+ Stage: Stage-15
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1 {key}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
+
+ Stage: Stage-10
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1 {key}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col4
+ Position of Big Table: 1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col0}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col4
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+
+PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest_j2
+POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest_j2
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j2
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-22_907_3066399957536170228/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j2
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-22_907_3066399957536170228/-mr-10000
+POSTHOOK: Lineage: dest_j2.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j2.value SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ]
+33815990627
Index: ql/src/test/results/clientpositive/auto_join12.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join12.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join12.q.out (revision 0)
@@ -0,0 +1,554 @@
+PREHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c2)))) src1) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c4)))) src2) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c3)) (< (. (TOK_TABLE_OR_COL src1) c1) 100))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key) c5) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) value) c6)))) src3) (AND (= (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src3) c5)) (< (. (TOK_TABLE_OR_COL src3) c5) 80)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL src1) c1) (. (TOK_TABLE_OR_COL src2) c4)))))))
+
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage , consists of Stage-9, Stage-10, Stage-11, Stage-1
+ Stage-9 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-9
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6, Stage-7
+ Stage-10 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-10
+ Stage-11 has a backup stage: Stage-1
+ Stage-7 depends on stages: Stage-11
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2:src
+ Fetch Operator
+ limit: -1
+ src3:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 0
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 80)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 80)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-45-03_043_1933998987408417444/-mr-10002
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1:src
+ Fetch Operator
+ limit: -1
+ src3:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 1
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 80)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 80)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 1
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-11
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1:src
+ Fetch Operator
+ limit: -1
+ src2:src
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 2
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ Position of Big Table: 2
+
+ Stage: Stage-7
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 80)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 80)
+ type: boolean
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {_col0}
+ 1 {_col1}
+ 2
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ 2 [Column[_col0]]
+ outputColumnNames: _col0, _col3
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 100)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 100)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ src2:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: string
+ src3:src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key < 80)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (_col0 < 80)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ condition expressions:
+ 0 {VALUE._col0}
+ 1 {VALUE._col1}
+ 2
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-03_632_3619370245375510201/-mr-10000
+POSTHOOK: query: SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-45-03_632_3619370245375510201/-mr-10000
+-136843922952
Index: ql/src/test/results/clientpositive/auto_join21.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join21.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join21.q.out (revision 0)
@@ -0,0 +1,2991 @@
+PREHOOK: query: explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_LEFTOUTERJOIN (TOK_TABREF src src1) (TOK_TABREF src src2) (AND (AND (= (. (TOK_TABLE_OR_COL src1) key) (. (TOK_TABLE_OR_COL src2) key)) (< (. (TOK_TABLE_OR_COL src1) key) 10)) (> (. (TOK_TABLE_OR_COL src2) key) 10))) (TOK_TABREF src src3) (AND (= (. (TOK_TABLE_OR_COL src2) key) (. (TOK_TABLE_OR_COL src3) key)) (< (. (TOK_TABLE_OR_COL src3) key) 10)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src1) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src1) value)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src2) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src2) value)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src3) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL src3) value)))))
+
+STAGE DEPENDENCIES:
+ Stage-7 is a root stage , consists of Stage-8, Stage-9, Stage-1
+ Stage-8 has a backup stage: Stage-1
+ Stage-5 depends on stages: Stage-8
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
+ Stage-9 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-9
+ Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src2
+ Fetch Operator
+ limit: -1
+ src3
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 0
+ src3
+ TableScan
+ alias: src3
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 0
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-47-54_368_8618712082979412893/-mr-10002
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ sort order: ++++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ src1
+ Fetch Operator
+ limit: -1
+ src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ Position of Big Table: 2
+
+ Stage: Stage-6
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src3
+ TableScan
+ alias: src3
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0 {(key < 10)}
+ 1
+ 2 {(key < 10)}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ 2 [Column[key]]
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Position of Big Table: 2
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src1
+ TableScan
+ alias: src1
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key > 10)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src3
+ TableScan
+ alias: src3
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 2
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join1 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ 2 {VALUE._col0} {VALUE._col1}
+ filter predicates:
+ 0 {(VALUE._col0 < 10)}
+ 1
+ 2 {(VALUE._col0 < 10)}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8, _col9
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: string
+ expr: _col8
+ type: string
+ expr: _col9
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-54_744_88626681810152975/-mr-10000
+POSTHOOK: query: SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-47-54_744_88626681810152975/-mr-10000
+NULL NULL NULL NULL 0 val_0
+NULL NULL NULL NULL 0 val_0
+NULL NULL NULL NULL 0 val_0
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 100 val_100
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 103 val_103
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 104 val_104
+NULL NULL NULL NULL 105 val_105
+NULL NULL NULL NULL 11 val_11
+NULL NULL NULL NULL 111 val_111
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 113 val_113
+NULL NULL NULL NULL 114 val_114
+NULL NULL NULL NULL 116 val_116
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 118 val_118
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 119 val_119
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 12 val_12
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 120 val_120
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 125 val_125
+NULL NULL NULL NULL 126 val_126
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 128 val_128
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 129 val_129
+NULL NULL NULL NULL 131 val_131
+NULL NULL NULL NULL 133 val_133
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 134 val_134
+NULL NULL NULL NULL 136 val_136
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 137 val_137
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 138 val_138
+NULL NULL NULL NULL 143 val_143
+NULL NULL NULL NULL 145 val_145
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 146 val_146
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 149 val_149
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 15 val_15
+NULL NULL NULL NULL 150 val_150
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 152 val_152
+NULL NULL NULL NULL 153 val_153
+NULL NULL NULL NULL 155 val_155
+NULL NULL NULL NULL 156 val_156
+NULL NULL NULL NULL 157 val_157
+NULL NULL NULL NULL 158 val_158
+NULL NULL NULL NULL 160 val_160
+NULL NULL NULL NULL 162 val_162
+NULL NULL NULL NULL 163 val_163
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 164 val_164
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 165 val_165
+NULL NULL NULL NULL 166 val_166
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 167 val_167
+NULL NULL NULL NULL 168 val_168
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 169 val_169
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 170 val_170
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 172 val_172
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 174 val_174
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 175 val_175
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 176 val_176
+NULL NULL NULL NULL 177 val_177
+NULL NULL NULL NULL 178 val_178
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 179 val_179
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 18 val_18
+NULL NULL NULL NULL 180 val_180
+NULL NULL NULL NULL 181 val_181
+NULL NULL NULL NULL 183 val_183
+NULL NULL NULL NULL 186 val_186
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 187 val_187
+NULL NULL NULL NULL 189 val_189
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 190 val_190
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 191 val_191
+NULL NULL NULL NULL 192 val_192
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 193 val_193
+NULL NULL NULL NULL 194 val_194
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 195 val_195
+NULL NULL NULL NULL 196 val_196
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 197 val_197
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 199 val_199
+NULL NULL NULL NULL 2 val_2
+NULL NULL NULL NULL 20 val_20
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 200 val_200
+NULL NULL NULL NULL 201 val_201
+NULL NULL NULL NULL 202 val_202
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 203 val_203
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 205 val_205
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 207 val_207
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 208 val_208
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 209 val_209
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 213 val_213
+NULL NULL NULL NULL 214 val_214
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 216 val_216
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 217 val_217
+NULL NULL NULL NULL 218 val_218
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 219 val_219
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 221 val_221
+NULL NULL NULL NULL 222 val_222
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 223 val_223
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 224 val_224
+NULL NULL NULL NULL 226 val_226
+NULL NULL NULL NULL 228 val_228
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 229 val_229
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 230 val_230
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 233 val_233
+NULL NULL NULL NULL 235 val_235
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 237 val_237
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 238 val_238
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 239 val_239
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 24 val_24
+NULL NULL NULL NULL 241 val_241
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 242 val_242
+NULL NULL NULL NULL 244 val_244
+NULL NULL NULL NULL 247 val_247
+NULL NULL NULL NULL 248 val_248
+NULL NULL NULL NULL 249 val_249
+NULL NULL NULL NULL 252 val_252
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 255 val_255
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 256 val_256
+NULL NULL NULL NULL 257 val_257
+NULL NULL NULL NULL 258 val_258
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 26 val_26
+NULL NULL NULL NULL 260 val_260
+NULL NULL NULL NULL 262 val_262
+NULL NULL NULL NULL 263 val_263
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 265 val_265
+NULL NULL NULL NULL 266 val_266
+NULL NULL NULL NULL 27 val_27
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 272 val_272
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 273 val_273
+NULL NULL NULL NULL 274 val_274
+NULL NULL NULL NULL 275 val_275
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 277 val_277
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 278 val_278
+NULL NULL NULL NULL 28 val_28
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 280 val_280
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 281 val_281
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 282 val_282
+NULL NULL NULL NULL 283 val_283
+NULL NULL NULL NULL 284 val_284
+NULL NULL NULL NULL 285 val_285
+NULL NULL NULL NULL 286 val_286
+NULL NULL NULL NULL 287 val_287
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 288 val_288
+NULL NULL NULL NULL 289 val_289
+NULL NULL NULL NULL 291 val_291
+NULL NULL NULL NULL 292 val_292
+NULL NULL NULL NULL 296 val_296
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 298 val_298
+NULL NULL NULL NULL 30 val_30
+NULL NULL NULL NULL 302 val_302
+NULL NULL NULL NULL 305 val_305
+NULL NULL NULL NULL 306 val_306
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 307 val_307
+NULL NULL NULL NULL 308 val_308
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 309 val_309
+NULL NULL NULL NULL 310 val_310
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 311 val_311
+NULL NULL NULL NULL 315 val_315
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 316 val_316
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 317 val_317
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 318 val_318
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 321 val_321
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 322 val_322
+NULL NULL NULL NULL 323 val_323
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 325 val_325
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 327 val_327
+NULL NULL NULL NULL 33 val_33
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 331 val_331
+NULL NULL NULL NULL 332 val_332
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 333 val_333
+NULL NULL NULL NULL 335 val_335
+NULL NULL NULL NULL 336 val_336
+NULL NULL NULL NULL 338 val_338
+NULL NULL NULL NULL 339 val_339
+NULL NULL NULL NULL 34 val_34
+NULL NULL NULL NULL 341 val_341
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 342 val_342
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 344 val_344
+NULL NULL NULL NULL 345 val_345
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 348 val_348
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 35 val_35
+NULL NULL NULL NULL 351 val_351
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 353 val_353
+NULL NULL NULL NULL 356 val_356
+NULL NULL NULL NULL 360 val_360
+NULL NULL NULL NULL 362 val_362
+NULL NULL NULL NULL 364 val_364
+NULL NULL NULL NULL 365 val_365
+NULL NULL NULL NULL 366 val_366
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 367 val_367
+NULL NULL NULL NULL 368 val_368
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 369 val_369
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 37 val_37
+NULL NULL NULL NULL 373 val_373
+NULL NULL NULL NULL 374 val_374
+NULL NULL NULL NULL 375 val_375
+NULL NULL NULL NULL 377 val_377
+NULL NULL NULL NULL 378 val_378
+NULL NULL NULL NULL 379 val_379
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 382 val_382
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 384 val_384
+NULL NULL NULL NULL 386 val_386
+NULL NULL NULL NULL 389 val_389
+NULL NULL NULL NULL 392 val_392
+NULL NULL NULL NULL 393 val_393
+NULL NULL NULL NULL 394 val_394
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 395 val_395
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 396 val_396
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 397 val_397
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 399 val_399
+NULL NULL NULL NULL 4 val_4
+NULL NULL NULL NULL 400 val_400
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 401 val_401
+NULL NULL NULL NULL 402 val_402
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 403 val_403
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 404 val_404
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 406 val_406
+NULL NULL NULL NULL 407 val_407
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 409 val_409
+NULL NULL NULL NULL 41 val_41
+NULL NULL NULL NULL 411 val_411
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 413 val_413
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 414 val_414
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 417 val_417
+NULL NULL NULL NULL 418 val_418
+NULL NULL NULL NULL 419 val_419
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 42 val_42
+NULL NULL NULL NULL 421 val_421
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 424 val_424
+NULL NULL NULL NULL 427 val_427
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 429 val_429
+NULL NULL NULL NULL 43 val_43
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 430 val_430
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 431 val_431
+NULL NULL NULL NULL 432 val_432
+NULL NULL NULL NULL 435 val_435
+NULL NULL NULL NULL 436 val_436
+NULL NULL NULL NULL 437 val_437
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 438 val_438
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 439 val_439
+NULL NULL NULL NULL 44 val_44
+NULL NULL NULL NULL 443 val_443
+NULL NULL NULL NULL 444 val_444
+NULL NULL NULL NULL 446 val_446
+NULL NULL NULL NULL 448 val_448
+NULL NULL NULL NULL 449 val_449
+NULL NULL NULL NULL 452 val_452
+NULL NULL NULL NULL 453 val_453
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 454 val_454
+NULL NULL NULL NULL 455 val_455
+NULL NULL NULL NULL 457 val_457
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 458 val_458
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 459 val_459
+NULL NULL NULL NULL 460 val_460
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 462 val_462
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 463 val_463
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 466 val_466
+NULL NULL NULL NULL 467 val_467
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 468 val_468
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 469 val_469
+NULL NULL NULL NULL 47 val_47
+NULL NULL NULL NULL 470 val_470
+NULL NULL NULL NULL 472 val_472
+NULL NULL NULL NULL 475 val_475
+NULL NULL NULL NULL 477 val_477
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 478 val_478
+NULL NULL NULL NULL 479 val_479
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 480 val_480
+NULL NULL NULL NULL 481 val_481
+NULL NULL NULL NULL 482 val_482
+NULL NULL NULL NULL 483 val_483
+NULL NULL NULL NULL 484 val_484
+NULL NULL NULL NULL 485 val_485
+NULL NULL NULL NULL 487 val_487
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 489 val_489
+NULL NULL NULL NULL 490 val_490
+NULL NULL NULL NULL 491 val_491
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 492 val_492
+NULL NULL NULL NULL 493 val_493
+NULL NULL NULL NULL 494 val_494
+NULL NULL NULL NULL 495 val_495
+NULL NULL NULL NULL 496 val_496
+NULL NULL NULL NULL 497 val_497
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 498 val_498
+NULL NULL NULL NULL 5 val_5
+NULL NULL NULL NULL 5 val_5
+NULL NULL NULL NULL 5 val_5
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 51 val_51
+NULL NULL NULL NULL 53 val_53
+NULL NULL NULL NULL 54 val_54
+NULL NULL NULL NULL 57 val_57
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 58 val_58
+NULL NULL NULL NULL 64 val_64
+NULL NULL NULL NULL 65 val_65
+NULL NULL NULL NULL 66 val_66
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 67 val_67
+NULL NULL NULL NULL 69 val_69
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 70 val_70
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 72 val_72
+NULL NULL NULL NULL 74 val_74
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 76 val_76
+NULL NULL NULL NULL 77 val_77
+NULL NULL NULL NULL 78 val_78
+NULL NULL NULL NULL 8 val_8
+NULL NULL NULL NULL 80 val_80
+NULL NULL NULL NULL 82 val_82
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 83 val_83
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 84 val_84
+NULL NULL NULL NULL 85 val_85
+NULL NULL NULL NULL 86 val_86
+NULL NULL NULL NULL 87 val_87
+NULL NULL NULL NULL 9 val_9
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 90 val_90
+NULL NULL NULL NULL 92 val_92
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 95 val_95
+NULL NULL NULL NULL 96 val_96
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 97 val_97
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
+NULL NULL NULL NULL 98 val_98
Index: ql/src/test/results/clientpositive/auto_join8.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join8.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join8.q.out (revision 0)
@@ -0,0 +1,351 @@
+PREHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) value) c2)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src1) key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) value) c4)) (TOK_WHERE (and (> (. (TOK_TABLE_OR_COL src2) key) 15) (< (. (TOK_TABLE_OR_COL src2) key) 25))))) b) (= (. (TOK_TABLE_OR_COL a) c1) (. (TOK_TABLE_OR_COL b) c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c1) c1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) c2) c2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c3) c3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c1)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c3)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) c4))) (TOK_WHERE (AND (TOK_FUNCTION TOK_ISNULL (. (TOK_TABLE_OR_COL c) c3)) (TOK_FUNCTION TOK_ISNOTNULL (. (TOK_TABLE_OR_COL c) c1))))))
+
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage , consists of Stage-6, Stage-1
+ Stage-6 has a backup stage: Stage-1
+ Stage-4 depends on stages: Stage-6
+ Stage-0 depends on stages: Stage-1, Stage-4
+ Stage-2 depends on stages: Stage-0
+ Stage-1
+
+STAGE PLANS:
+ Stage: Stage-5
+ Conditional Operator
+
+ Stage: Stage-6
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ c:b:src2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[_col0]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Filter Operator
+ predicate:
+ expr: (_col2 is null and _col0 is not null)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ c:a:src1
+ TableScan
+ alias: src1
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 10) and (key < 20))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ c:b:src2
+ TableScan
+ alias: src2
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: ((key > 15) and (key < 25))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Filter Operator
+ predicate:
+ expr: (_col2 is null and _col0 is not null)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: UDFToInteger(_col0)
+ type: int
+ expr: _col1
+ type: string
+ expr: UDFToInteger(_col2)
+ type: int
+ expr: _col3
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-35_928_3795245960359144388/-mr-10000
+POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-50-35_928_3795245960359144388/-mr-10000
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+-7158439905
Index: ql/src/test/results/clientpositive/auto_join18.q.out
===================================================================
--- ql/src/test/results/clientpositive/auto_join18.q.out (revision 0)
+++ ql/src/test/results/clientpositive/auto_join18.q.out (revision 0)
@@ -0,0 +1,288 @@
+PREHOOK: query: explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src1) key) key) (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src1) value)) value)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src1) key)))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src1 src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src2) key) key) (TOK_SELEXPR (TOK_FUNCTIONDI count (. (TOK_TABLE_OR_COL src2) value)) value)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src2) key)))) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION hash (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL a) value) (. (TOK_TABLE_OR_COL b) key) (. (TOK_TABLE_OR_COL b) value)))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-3 depends on stages: Stage-2
+ Stage-4 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ b:src2
+ TableScan
+ alias: src2
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col2
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT KEY._col1:0._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ $INTNAME1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Outer Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1 {VALUE._col0} {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: bigint
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Group By Operator
+ aggregations:
+ expr: sum(hash(_col0,_col1,_col2,_col3))
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/liyintang/hive_2010-11-15_16-46-23_684_8729096496625979401/-mr-10003
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: sum(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ a:src1
+ TableScan
+ alias: src1
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ aggregations:
+ expr: count(value)
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-23_849_2497457649667386022/-mr-10000
+POSTHOOK: query: SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-15_16-46-23_849_2497457649667386022/-mr-10000
+379685492277
Index: ql/src/test/queries/clientpositive/auto_join18_multi_distinct.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join18_multi_distinct.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join18_multi_distinct.q (revision 0)
@@ -0,0 +1,29 @@
+set hive.auto.convert.join = true;
+
+explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key);
+
+
+ SELECT sum(hash(a.key, a.value, b.key, b.value1, b.value2))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value1,
+ count(distinct(src2.key)) AS value2
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key);
Index: ql/src/test/queries/clientpositive/auto_join10.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join10.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join10.q (revision 0)
@@ -0,0 +1,17 @@
+set hive.auto.convert.join = true;
+
+explain
+FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT src.* FROM src) x
+JOIN
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
Index: ql/src/test/queries/clientpositive/auto_join11.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join11.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join11.q (revision 0)
@@ -0,0 +1,16 @@
+set hive.auto.convert.join = true;
+
+explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100;
+
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100;
Index: ql/src/test/queries/clientpositive/auto_join20.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join20.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join20.q (revision 0)
@@ -0,0 +1,31 @@
+set hive.auto.convert.join = true;
+
+explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a;
+
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a;
+
+explain
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a;
+
+select sum(hash(a.k1,a.v1,a.k2,a.v2,a.k3,a.v3))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 , src3.key as k3, src3.value as v3
+FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
+SORT BY k1,v1,k2,v2,k3,v3
+)a;
Index: ql/src/test/queries/clientpositive/auto_join0.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join0.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join0.q (revision 0)
@@ -0,0 +1,23 @@
+
+set hive.auto.convert.join = true;
+
+explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a;
+
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1,
+ src2.key as k2, src2.value as v2 FROM
+ (SELECT * FROM src WHERE src.key < 10) src1
+ JOIN
+ (SELECT * FROM src WHERE src.key < 10) src2
+ SORT BY k1, v1, k2, v2
+) a;
Index: ql/src/test/queries/clientpositive/auto_join12.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join12.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join12.q (revision 0)
@@ -0,0 +1,25 @@
+
+
+set hive.auto.convert.join = true;
+
+
+explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80;
+
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 = src3.c5 AND src3.c5 < 80;
Index: ql/src/test/queries/clientpositive/auto_join21.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join21.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join21.q (revision 0)
@@ -0,0 +1,5 @@
+set hive.auto.convert.join = true;
+explain
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value;
+
+SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value;
Index: ql/src/test/queries/clientpositive/auto_join1.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join1.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join1.q (revision 0)
@@ -0,0 +1,12 @@
+set hive.auto.convert.join =true;
+
+CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value;
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value;
+
+SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1;
\ No newline at end of file
Index: ql/src/test/queries/clientpositive/auto_join13.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join13.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join13.q (revision 0)
@@ -0,0 +1,23 @@
+
+set hive.auto.convert.join = true;
+
+explain
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200;
+
+SELECT sum(hash(src1.c1, src2.c4))
+FROM
+(SELECT src.key as c1, src.value as c2 from src) src1
+JOIN
+(SELECT src.key as c3, src.value as c4 from src) src2
+ON src1.c1 = src2.c3 AND src1.c1 < 100
+JOIN
+(SELECT src.key as c5, src.value as c6 from src) src3
+ON src1.c1 + src2.c3 = src3.c5 AND src3.c5 < 200;
Index: ql/src/test/queries/clientpositive/auto_join22.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join22.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join22.q (revision 0)
@@ -0,0 +1,5 @@
+set hive.auto.convert.join = true;
+explain
+SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5;
+
+SELECT sum(hash(src5.src1_value)) FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5;
Index: ql/src/test/queries/clientpositive/auto_join2.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join2.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join2.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value;
+
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value;
+
+SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2;
Index: ql/src/test/queries/clientpositive/auto_join14.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join14.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join14.q (revision 0)
@@ -0,0 +1,16 @@
+
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE;
+
+set mapred.job.tracker=does.notexist.com:666;
+set hive.exec.mode.local.auto=true;
+
+explain
+FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value;
+
+FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
+INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value;
+
+SELECT sum(hash(dest1.c1,dest1.c2)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join23.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join23.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join23.q (revision 0)
@@ -0,0 +1,6 @@
+set hive.auto.convert.join = true;
+
+explain
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value;
+
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10 SORT BY src1.key, src1.value, src2.key, src2.value;
Index: ql/src/test/queries/clientpositive/auto_join3.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join3.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join3.q (revision 0)
@@ -0,0 +1,12 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value;
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value;
+
+SELECT sum(hash(dest1.key,dest1.value)) FROM dest1;
\ No newline at end of file
Index: ql/src/test/queries/clientpositive/auto_join15.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join15.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join15.q (revision 0)
@@ -0,0 +1,19 @@
+
+set hive.auto.convert.join = true;
+
+explain
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a;
+
+
+select sum(hash(a.k1,a.v1,a.k2, a.v2))
+from (
+SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+SORT BY k1, v1, k2, v2
+) a;
+
Index: ql/src/test/queries/clientpositive/auto_join24.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join24.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join24.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.auto.convert.join = true;
+
+create table tst1(key STRING, cnt INT);
+
+INSERT OVERWRITE TABLE tst1
+SELECT a.key, count(1) FROM src a group by a.key;
+
+explain
+SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key;
+
+SELECT sum(a.cnt) FROM tst1 a JOIN tst1 b ON a.key = b.key;
+
+
Index: ql/src/test/queries/clientpositive/auto_join4.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join4.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join4.q (revision 0)
@@ -0,0 +1,34 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE;
+
+explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join16.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join16.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join16.q (revision 0)
@@ -0,0 +1,18 @@
+
+set hive.auto.convert.join = true;
+
+
+explain
+SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200;
+
+SELECT sum(hash(subq.key, tab.value))
+FROM
+(select a.key, a.value from src a where a.key > 10 ) subq
+JOIN src tab
+ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value)
+where tab.value < 200;
\ No newline at end of file
Index: ql/src/test/queries/clientpositive/auto_join25.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join25.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join25.q (revision 0)
@@ -0,0 +1,28 @@
+set hive.auto.convert.join = true;
+set hive.mapjoin.localtask.max.memory.usage = 0.0001;
+set hive.mapjoin.check.memory.rows = 2;
+
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11');
+
+SELECT sum(hash(dest1.key,dest1.value)) FROM dest1;
+
+
+
+CREATE TABLE dest_j2(key INT, value STRING) STORED AS TEXTFILE;
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key)
+INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value;
+
+SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2;
+
+CREATE TABLE dest_j1(key INT, value STRING) STORED AS TEXTFILE;
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value;
+
+SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1;
+
Index: ql/src/test/queries/clientpositive/auto_join5.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join5.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join5.q (revision 0)
@@ -0,0 +1,34 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE;
+
+explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ RIGHT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join17.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join17.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join17.q (revision 0)
@@ -0,0 +1,14 @@
+
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(key1 INT, value1 STRING, key2 INT, value2 STRING) STORED AS TEXTFILE;
+
+explain
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*;
+
+
+FROM src src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.*;
+
+SELECT sum(hash(dest1.key1,dest1.value1,dest1.key2,dest1.value2)) FROM dest1;
\ No newline at end of file
Index: ql/src/test/queries/clientpositive/auto_join_nulls.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join_nulls.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join_nulls.q (revision 0)
@@ -0,0 +1,29 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE myinput1(key int, value int);
+LOAD DATA LOCAL INPATH '../data/files/in1.txt' INTO TABLE myinput1;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value;
+
Index: ql/src/test/queries/clientpositive/auto_join6.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join6.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join6.q (revision 0)
@@ -0,0 +1,35 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE;
+
+explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4;
+
+
+SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join18.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join18.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join18.q (revision 0)
@@ -0,0 +1,27 @@
+
+set hive.auto.convert.join = true;
+explain
+ SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key);
+
+
+ SELECT sum(hash(a.key, a.value, b.key, b.value))
+ FROM
+ (
+ SELECT src1.key as key, count(src1.value) AS value FROM src src1 group by src1.key
+ ) a
+ FULL OUTER JOIN
+ (
+ SELECT src2.key as key, count(distinct(src2.value)) AS value
+ FROM src1 src2 group by src2.key
+ ) b
+ ON (a.key = b.key);
Index: ql/src/test/queries/clientpositive/auto_join7.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join7.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join7.q (revision 0)
@@ -0,0 +1,46 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING) STORED AS TEXTFILE;
+
+
+explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6;
+
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ FULL OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ LEFT OUTER JOIN
+ (
+ FROM src src3 SELECT src3.key AS c5, src3.value AS c6 WHERE src3.key > 20 and src3.key < 25
+ ) c
+ ON (a.c1 = c.c5)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6;
+
+
+SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4,dest1.c5,dest1.c6)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join19.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join19.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join19.q (revision 0)
@@ -0,0 +1,16 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11');
+
+
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value
+where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11');
+
+
+SELECT sum(hash(dest1.key,dest1.value)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join8.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join8.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join8.q (revision 0)
@@ -0,0 +1,34 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE;
+
+explain
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL;
+
+FROM (
+ FROM
+ (
+ FROM src src1 SELECT src1.key AS c1, src1.value AS c2 WHERE src1.key > 10 and src1.key < 20
+ ) a
+ LEFT OUTER JOIN
+ (
+ FROM src src2 SELECT src2.key AS c3, src2.value AS c4 WHERE src2.key > 15 and src2.key < 25
+ ) b
+ ON (a.c1 = b.c3)
+ SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4
+) c
+INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL;
+
+SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join9.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join9.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join9.q (revision 0)
@@ -0,0 +1,14 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
+
+explain
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12';
+
+FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key)
+INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12';
+
+
+
+SELECT sum(hash(dest1.key,dest1.value)) FROM dest1;
Index: ql/src/test/queries/clientpositive/auto_join_filters.q
===================================================================
--- ql/src/test/queries/clientpositive/auto_join_filters.q (revision 0)
+++ ql/src/test/queries/clientpositive/auto_join_filters.q (revision 0)
@@ -0,0 +1,82 @@
+set hive.auto.convert.join = true;
+
+CREATE TABLE myinput1(key int, value int);
+LOAD DATA LOCAL INPATH '../data/files/in3.txt' INTO TABLE myinput1;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+
+
+CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS;
+CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS;
+LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input1;
+LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input1;
+LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input2;
+LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input2;
+
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+SET hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+
+SET hive.outerjoin.supports.filters = false;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java (working copy)
@@ -205,7 +205,7 @@
Operator extends Serializable> input = parentOp.get(0);
input.getChildOperators().clear();
- RowResolver inputRR = pGraphContext.getOpParseCtx().get(input).getRR();
+ RowResolver inputRR = pGraphContext.getOpParseCtx().get(input).getRowResolver();
ArrayList exprs = new ArrayList();
ArrayList outputs = new ArrayList();
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (working copy)
@@ -179,7 +179,7 @@
cppCtx.getPrunedColLists().put((Operator extends Serializable>) nd,
cols);
ArrayList needed_columns = new ArrayList();
- RowResolver inputRR = cppCtx.getOpToParseCtxMap().get(scanOp).getRR();
+ RowResolver inputRR = cppCtx.getOpToParseCtxMap().get(scanOp).getRowResolver();
TableScanDesc desc = scanOp.getConf();
List virtualCols = desc.getVirtualCols();
List newVirtualCols = new ArrayList();
@@ -232,7 +232,7 @@
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
HashMap, OpParseContext> opToParseCtxMap = cppCtx
.getOpToParseCtxMap();
- RowResolver redSinkRR = opToParseCtxMap.get(op).getRR();
+ RowResolver redSinkRR = opToParseCtxMap.get(op).getRowResolver();
ReduceSinkDesc conf = op.getConf();
List> childOperators = op
.getChildOperators();
@@ -250,7 +250,7 @@
assert parentOperators.size() == 1;
Operator extends Serializable> par = parentOperators.get(0);
JoinOperator childJoin = (JoinOperator) childOperators.get(0);
- RowResolver parRR = opToParseCtxMap.get(par).getRR();
+ RowResolver parRR = opToParseCtxMap.get(par).getRowResolver();
List childJoinCols = cppCtx.getJoinPrunedColLists().get(
childJoin).get((byte) conf.getTag());
boolean[] flags = new boolean[conf.getValueCols().size()];
@@ -383,7 +383,7 @@
ArrayList newOutputColumnNames = new ArrayList();
ArrayList rs_oldsignature = op.getSchema().getSignature();
ArrayList rs_newsignature = new ArrayList();
- RowResolver old_rr = cppCtx.getOpToParseCtxMap().get(op).getRR();
+ RowResolver old_rr = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
RowResolver new_rr = new RowResolver();
for (String col : cols) {
int index = originalOutputColumnNames.indexOf(col);
@@ -394,7 +394,7 @@
ColumnInfo columnInfo = old_rr.get(tabcol[0], tabcol[1]);
new_rr.put(tabcol[0], tabcol[1], columnInfo);
}
- cppCtx.getOpToParseCtxMap().get(op).setRR(new_rr);
+ cppCtx.getOpToParseCtxMap().get(op).setRowResolver(new_rr);
op.getSchema().setSignature(rs_newsignature);
conf.setColList(newColList);
conf.setOutputColumnNames(newOutputColumnNames);
@@ -465,7 +465,7 @@
Map oldMap = reduce.getColumnExprMap();
Map newMap = new HashMap();
ArrayList sig = new ArrayList();
- RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(reduce).getRR();
+ RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(reduce).getRowResolver();
RowResolver newRR = new RowResolver();
ArrayList originalValueOutputColNames = reduceConf
.getOutputValueColumnNames();
@@ -493,7 +493,7 @@
ArrayList keyCols = reduceConf.getKeyCols();
List keys = new ArrayList();
RowResolver parResover = cppCtx.getOpToParseCtxMap().get(
- reduce.getParentOperators().get(0)).getRR();
+ reduce.getParentOperators().get(0)).getRowResolver();
for (int i = 0; i < keyCols.size(); i++) {
keys = Utilities.mergeUniqElems(keys, keyCols.get(i).getCols());
}
@@ -506,7 +506,7 @@
}
}
- cppCtx.getOpToParseCtxMap().get(reduce).setRR(newRR);
+ cppCtx.getOpToParseCtxMap().get(reduce).setRowResolver(newRR);
reduce.setColumnExprMap(newMap);
reduce.getSchema().setSignature(sig);
reduceConf.setOutputValueColumnNames(newOutputColNames);
@@ -614,7 +614,7 @@
}
}
- RowResolver joinRR = cppCtx.getOpToParseCtxMap().get(op).getRR();
+ RowResolver joinRR = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
RowResolver newJoinRR = new RowResolver();
ArrayList outputCols = new ArrayList();
ArrayList rs = new ArrayList();
@@ -699,7 +699,7 @@
op.setColumnExprMap(newColExprMap);
conf.setOutputColumnNames(outputCols);
op.getSchema().setSignature(rs);
- cppCtx.getOpToParseCtxMap().get(op).setRR(newJoinRR);
+ cppCtx.getOpToParseCtxMap().get(op).setRowResolver(newJoinRR);
cppCtx.getJoinPrunedColLists().put(op, prunedColLists);
}
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy)
@@ -217,7 +217,7 @@
// Add the extract operator to get the value fields
RowResolver out_rwsch = new RowResolver();
- RowResolver interim_rwsch = ctx.getParseCtx().getOpParseCtx().get(fsOp).getRR();
+ RowResolver interim_rwsch = ctx.getParseCtx().getOpParseCtx().get(fsOp).getRowResolver();
Integer pos = Integer.valueOf(0);
for (ColumnInfo colInfo : interim_rwsch.getColumnInfos()) {
String[] info = interim_rwsch.reverseLookup(colInfo.getInternalName());
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (working copy)
@@ -858,8 +858,9 @@
// create a dummy tableScan operator on top of op
// TableScanOperator is implicitly created here for each MapOperator
+ RowResolver rowResolver = opProcCtx.getParseCtx().getOpParseCtx().get(parent).getRowResolver();
Operator extends Serializable> ts_op = putOpInsertMap(OperatorFactory
- .get(TableScanDesc.class, parent.getSchema()), null, parseCtx);
+ .get(TableScanDesc.class, parent.getSchema()), rowResolver, parseCtx);
childOpList = new ArrayList>();
childOpList.add(op);
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (working copy)
@@ -42,11 +42,16 @@
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolver;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin;
import org.apache.hadoop.hive.ql.plan.ConditionalWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx;
import org.apache.hadoop.hive.ql.plan.ConditionalResolverSkewJoin.ConditionalResolverSkewJoinCtx;
/**
@@ -60,7 +65,7 @@
//create dispatcher and graph walker
Dispatcher disp = new LocalMapJoinTaskDispatcher(pctx);
- GraphWalker ogw = new DefaultGraphWalker(disp);
+ TaskGraphWalker ogw = new TaskGraphWalker(disp);
//get all the tasks nodes from root task
ArrayList topNodes = new ArrayList();
@@ -106,6 +111,11 @@
MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork,
physicalContext.getParseContext().getConf());
+ //set the backup task from curr task
+ localTask.setBackupTask(currTask.getBackupTask());
+ localTask.setBackupChildrenTasks(currTask.getBackupChildrenTasks());
+ currTask.setBackupChildrenTasks(null);
+ currTask.setBackupTask(null);
//replace the map join operator to local_map_join operator in the operator tree
//and return all the dummy parent
@@ -149,31 +159,64 @@
listWork.set(index,(Serializable)localwork);
conditionalWork.setListWorks(listWork);
- //get bigKeysDirToTaskMap
- ConditionalResolverSkewJoinCtx context =
- (ConditionalResolverSkewJoinCtx) conditionalTask.getResolverCtx();
- HashMap> bigKeysDirToTaskMap =
- context.getDirToTaskMap();
+ ConditionalResolver resolver = conditionalTask.getResolver();
+ if(resolver instanceof ConditionalResolverSkewJoin){
+ //get bigKeysDirToTaskMap
+ ConditionalResolverSkewJoinCtx context =
+ (ConditionalResolverSkewJoinCtx) conditionalTask.getResolverCtx();
+ HashMap> bigKeysDirToTaskMap =
+ context.getDirToTaskMap();
- //to avoid concurrent modify the hashmap
- HashMap> newbigKeysDirToTaskMap =
- new HashMap>();
+ //to avoid concurrent modify the hashmap
+ HashMap> newbigKeysDirToTaskMap =
+ new HashMap>();
- //reset the resolver
- for(Map.Entry> entry: bigKeysDirToTaskMap.entrySet()){
- Task extends Serializable> task = entry.getValue();
- String key = entry.getKey();
+ //reset the resolver
+ for(Map.Entry> entry: bigKeysDirToTaskMap.entrySet()){
+ Task extends Serializable> task = entry.getValue();
+ String key = entry.getKey();
+ if(task.equals(currTask)){
+ newbigKeysDirToTaskMap.put(key, localTask);
+ }else{
+ newbigKeysDirToTaskMap.put(key, task);
+ }
+ }
- if(task.equals(currTask)){
- newbigKeysDirToTaskMap.put(key, localTask);
- }else{
- newbigKeysDirToTaskMap.put(key, task);
+ context.setDirToTaskMap(newbigKeysDirToTaskMap);
+ conditionalTask.setResolverCtx(context);
+
+ }else if(resolver instanceof ConditionalResolverCommonJoin){
+ //get bigKeysDirToTaskMap
+ ConditionalResolverCommonJoinCtx context =
+ (ConditionalResolverCommonJoinCtx) conditionalTask.getResolverCtx();
+ HashMap> aliasToWork =
+ context.getAliasToTask();
+
+ //to avoid concurrent modify the hashmap
+ HashMap> newAliasToWork =
+ new HashMap>();
+
+ //reset the resolver
+ for(Map.Entry> entry: aliasToWork.entrySet()){
+ Task extends Serializable> task = entry.getValue();
+ String key = entry.getKey();
+
+ if(task.equals(currTask)){
+ newAliasToWork.put(key, localTask);
+ }else{
+ newAliasToWork.put(key, task);
+ }
}
+
+ context.setAliasToTask(newAliasToWork);
+ conditionalTask.setResolverCtx(context);
+
+ }else{
+
}
- context.setDirToTaskMap(newbigKeysDirToTaskMap);
- conditionalTask.setResolverCtx(context);
+
}
}
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java (working copy)
@@ -49,6 +49,9 @@
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESKEWJOIN)) {
resolvers.add(new SkewJoinResolver());
}
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) {
+ resolvers.add(new CommonJoinResolver());
+ }
resolvers.add(new MapJoinResolver());
}
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (working copy)
@@ -273,7 +273,7 @@
JoinOperator cloneJoinOp = (JoinOperator) reducer;
MapJoinDesc mapJoinDescriptor = new MapJoinDesc(newJoinKeys, keyTblDesc,
- newJoinValues, newJoinValueTblDesc, joinDescriptor
+ newJoinValues, newJoinValueTblDesc, newJoinValueTblDesc,joinDescriptor
.getOutputColumnNames(), i, joinDescriptor.getConds(),
joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin());
mapJoinDescriptor.setTagOrder(tags);
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinResolver.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinResolver.java (revision 0)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinResolver.java (revision 0)
@@ -0,0 +1,226 @@
+package org.apache.hadoop.hive.ql.optimizer.physical;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.MapRedTask;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
+import org.apache.hadoop.hive.ql.lib.TaskGraphWalker.TaskGraphWalkerContext;
+import org.apache.hadoop.hive.ql.optimizer.MapJoinProcessor;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.QBJoinTree;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin;
+import org.apache.hadoop.hive.ql.plan.ConditionalWork;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.ConditionalResolverCommonJoin.ConditionalResolverCommonJoinCtx;
+
+
+public class CommonJoinResolver implements PhysicalPlanResolver {
+ @Override
+ public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
+
+ // create dispatcher and graph walker
+ Dispatcher disp = new CommonJoinTaskDispatcher(pctx);
+ TaskGraphWalker ogw = new TaskGraphWalker(disp);
+
+ // get all the tasks nodes from root task
+ ArrayList topNodes = new ArrayList();
+ topNodes.addAll(pctx.rootTasks);
+
+ // begin to walk through the task tree.
+ ogw.startWalking(topNodes, null);
+ return pctx;
+ }
+
+
+ /**
+ * Iterator each tasks. If this task has a local work,create a new task for this local work, named
+ * MapredLocalTask. then make this new generated task depends on current task's parent task, and
+ * make current task depends on this new generated task
+ */
+ class CommonJoinTaskDispatcher implements Dispatcher {
+
+ private final PhysicalContext physicalContext;
+
+ public CommonJoinTaskDispatcher(PhysicalContext context) {
+ super();
+ physicalContext = context;
+ }
+
+ private ConditionalTask processCurrentTask(MapRedTask currTask, ConditionalTask conditionalTask)
+ throws SemanticException {
+
+ // whether it contains common join op; if contains, return this common join op
+ JoinOperator joinOp = getJoinOp(currTask);
+ if (joinOp == null) {
+ return null;
+ }
+ MapredWork currWork = currTask.getWork();
+ // create conditional work list and task list
+ List listWorks = new ArrayList();
+ List> listTasks = new ArrayList>();
+
+ // create alias to task mapping and alias to input file mapping for resolver
+ HashMap> aliasToTask = new HashMap>();
+ HashMap aliasToPath = new HashMap();
+ HashMap> pathToAliases = currTask.getWork().getPathToAliases();
+
+ // get parseCtx for this Join Operator
+ ParseContext parseCtx = physicalContext.getParseContext();
+ QBJoinTree joinTree = parseCtx.getJoinContext().get(joinOp);
+
+ // start to generate multiple map join tasks
+ JoinDesc joinDesc = joinOp.getConf();
+ Byte[] order = joinDesc.getTagOrder();
+ int numAliases = order.length;
+
+ try {
+ HashSet smallTableOnlySet = MapJoinProcessor.getSmallTableOnlySet(joinDesc
+ .getConds());
+ // no table could be the big table; there is no need to convert
+ if (smallTableOnlySet == null) {
+ return null;
+ }
+ currWork.setOpParseCtxMap(parseCtx.getOpParseCtx());
+ currWork.setJoinTree(joinTree);
+
+ String xml = currWork.toXML();
+ String bigTableAlias = null;
+
+ if(smallTableOnlySet.size() == numAliases) {
+ return null;
+ }
+
+ for (int i = 0; i < numAliases; i++) {
+ // this table cannot be big table
+ if (smallTableOnlySet.contains(i)) {
+ continue;
+ }
+
+ // create map join task and set big table as i
+ // deep copy a new mapred work from xml
+ InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8"));
+ MapredWork newWork = Utilities.deserializeMapRedWork(in, physicalContext.getConf());
+ // create a mapred task for this work
+ MapRedTask newTask = (MapRedTask) TaskFactory.get(newWork, physicalContext
+ .getParseContext().getConf());
+ JoinOperator newJoinOp = getJoinOp(newTask);
+
+ // optimize this newWork and assume big table position is i
+ bigTableAlias = MapJoinProcessor.genMapJoinOpAndLocalWork(newWork, newJoinOp, i);
+
+ // add into conditional task
+ listWorks.add(newWork);
+ listTasks.add(newTask);
+
+ //set up backup task
+ newTask.setBackupTask(currTask);
+ newTask.setBackupChildrenTasks(currTask.getChildTasks());
+
+ // put the mapping alias to task
+ aliasToTask.put(bigTableAlias, newTask);
+
+ // set alias to path
+ for (Map.Entry> entry : pathToAliases.entrySet()) {
+ String path = entry.getKey();
+ ArrayList aliasList = entry.getValue();
+ if (aliasList.contains(bigTableAlias)) {
+ aliasToPath.put(bigTableAlias, path);
+ }
+ }
+
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new SemanticException("Generate Map Join Task Error: " + e.getMessage());
+ }
+
+ // insert current common join task to conditional task
+ listWorks.add(currTask.getWork());
+ listTasks.add(currTask);
+ // clear JoinTree and OP Parse Context
+ currWork.setOpParseCtxMap(null);
+ currWork.setJoinTree(null);
+
+ // create conditional task and insert conditional task into task tree
+ ConditionalWork cndWork = new ConditionalWork(listWorks);
+ ConditionalTask cndTsk = (ConditionalTask) TaskFactory.get(cndWork, parseCtx.getConf());
+ cndTsk.setListTasks(listTasks);
+
+ // set resolver and resolver context
+ cndTsk.setResolver(new ConditionalResolverCommonJoin());
+ ConditionalResolverCommonJoinCtx resolverCtx = new ConditionalResolverCommonJoinCtx();
+ resolverCtx.setAliasToPath(aliasToPath);
+ resolverCtx.setAliasToTask(aliasToTask);
+ resolverCtx.setCommonJoinTask(currTask);
+ cndTsk.setResolverCtx(resolverCtx);
+
+ ((Task extends Serializable>) currTask).replaceWithConditionalTask(cndTsk, physicalContext);
+ return cndTsk;
+ }
+
+
+ @Override
+ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs)
+ throws SemanticException {
+ if (nodeOutputs == null || nodeOutputs.length == 0) {
+ throw new SemanticException("No Dispatch Context");
+ }
+
+ TaskGraphWalkerContext walkerCtx = (TaskGraphWalkerContext) nodeOutputs[0];
+
+ Task extends Serializable> currTask = (Task extends Serializable>) nd;
+ // not map reduce task or not conditional task, just skip
+ if (currTask.isMapRedTask()) {
+ if (currTask instanceof ConditionalTask) {
+ // get the list of task
+ List> taskList = ((ConditionalTask) currTask).getListTasks();
+ for (Task extends Serializable> tsk : taskList) {
+ if (tsk.isMapRedTask()) {
+ ConditionalTask cndTask = this.processCurrentTask((MapRedTask) tsk,
+ ((ConditionalTask) currTask));
+ walkerCtx.addToDispatchList(cndTask);
+ }
+ }
+ } else {
+ ConditionalTask cndTask = this.processCurrentTask((MapRedTask) currTask, null);
+ walkerCtx.addToDispatchList(cndTask);
+ }
+ }
+ return null;
+ }
+
+
+ private JoinOperator getJoinOp(MapRedTask task) throws SemanticException {
+ if (task.getWork() == null) {
+ return null;
+ }
+
+ Operator extends Serializable> reducerOp = task.getWork().getReducer();
+ if (reducerOp instanceof JoinOperator) {
+ return (JoinOperator) reducerOp;
+ } else {
+ return null;
+ }
+
+ }
+ }
+}
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (working copy)
@@ -21,6 +21,7 @@
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
@@ -44,6 +45,7 @@
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.ScriptOperator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -53,7 +55,6 @@
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.GenMapRedWalker;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
@@ -61,23 +62,28 @@
import org.apache.hadoop.hive.ql.parse.QBJoinTree;
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
-import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.FetchWork;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
/**
- * Implementation of one of the rule-based map join optimization. User passes
- * hints to specify map-joins and during this optimization, all user specified
- * map joins are converted to MapJoins - the reduce sink operator above the join
- * are converted to map sink operators. In future, once statistics are
- * implemented, this transformation can also be done based on costs.
+ * Implementation of one of the rule-based map join optimization. User passes hints to specify
+ * map-joins and during this optimization, all user specified map joins are converted to MapJoins -
+ * the reduce sink operator above the join are converted to map sink operators. In future, once
+ * statistics are implemented, this transformation can also be done based on costs.
*/
public class MapJoinProcessor implements Transform {
@@ -93,14 +99,160 @@
}
@SuppressWarnings("nls")
- private Operator extends Serializable> putOpInsertMap(
- Operator extends Serializable> op, RowResolver rr) {
+ private Operator extends Serializable> putOpInsertMap(Operator extends Serializable> op,
+ RowResolver rr) {
OpParseContext ctx = new OpParseContext(rr);
pGraphContext.getOpParseCtx().put(op, ctx);
return op;
}
/**
+ * Generate the MapRed Local Work
+ * @param newWork
+ * @param mapJoinOp
+ * @param bigTablePos
+ * @return
+ * @throws SemanticException
+ */
+ private static String genMapJoinLocalWork(MapredWork newWork, MapJoinOperator mapJoinOp,
+ int bigTablePos) throws SemanticException {
+ // keep the small table alias to avoid concurrent modification exception
+ ArrayList smallTableAliasList = new ArrayList();
+ String bigTableAlias = null;
+
+ // create a new MapredLocalWork
+ MapredLocalWork newLocalWork = new MapredLocalWork(
+ new LinkedHashMap>(),
+ new LinkedHashMap());
+
+ for (Map.Entry> entry : newWork.getAliasToWork()
+ .entrySet()) {
+ String alias = entry.getKey();
+ Operator extends Serializable> op = entry.getValue();
+ // get table scan op
+ if (!(op instanceof TableScanOperator)) {
+ throw new SemanticException("top op is not table scan");
+ }
+ TableScanOperator tableScanOp = (TableScanOperator) op;
+
+ // if the table scan is for big table; then skip it
+ // tracing down the operator tree from the table scan operator
+ Operator extends Serializable> parentOp = tableScanOp;
+ Operator extends Serializable> childOp = tableScanOp.getChildOperators().get(0);
+ while ((childOp != null) && (!childOp.equals(mapJoinOp))) {
+ parentOp = childOp;
+ assert parentOp.getChildOperators().size() == 1;
+ childOp = parentOp.getChildOperators().get(0);
+ }
+ if (childOp == null) {
+ throw new SemanticException(
+ "Cannot find join op by tracing down the table scan operator tree");
+ }
+ // skip the big table pos
+ int i = childOp.getParentOperators().indexOf(parentOp);
+ if (i == bigTablePos) {
+ bigTableAlias = alias;
+ continue;
+ }
+ // set alias to work and put into smallTableAliasList
+ newLocalWork.getAliasToWork().put(alias, tableScanOp);
+ smallTableAliasList.add(alias);
+ // get input path and remove this alias from pathToAlias
+ // because this file will be fetched by fetch operator
+ LinkedHashMap> pathToAliases = newWork.getPathToAliases();
+
+ // keep record all the input path for this alias
+ HashSet pathSet = new HashSet();
+ HashSet emptyPath = new HashSet();
+ for (Map.Entry> entry2 : pathToAliases.entrySet()) {
+ String path = entry2.getKey();
+ ArrayList list = entry2.getValue();
+ if (list.contains(alias)) {
+ // add to path set
+ if (!pathSet.contains(path)) {
+ pathSet.add(path);
+ }
+ //remove this alias from the alias list
+ list.remove(alias);
+ if(list.size() == 0) {
+ emptyPath.add(path);
+ }
+ }
+ }
+ //remove the path, with which no alias associates
+ for (String path : emptyPath) {
+ pathToAliases.remove(path);
+ }
+
+ if (pathSet.size() == 0) {
+ throw new SemanticException("No input path for alias " + alias);
+ }
+
+ // create fetch work
+ FetchWork fetchWork = null;
+ List partDir = new ArrayList();
+ List partDesc = new ArrayList();
+
+ for (String tablePath : pathSet) {
+ PartitionDesc partitionDesc = newWork.getPathToPartitionInfo().get(tablePath);
+ // create fetchwork for non partitioned table
+ if (partitionDesc.getPartSpec() == null || partitionDesc.getPartSpec().size() == 0) {
+ fetchWork = new FetchWork(tablePath, partitionDesc.getTableDesc());
+ break;
+ }
+ // if table is partitioned,add partDir and partitionDesc
+ partDir.add(tablePath);
+ partDesc.add(partitionDesc);
+ }
+ // create fetchwork for partitioned table
+ if (fetchWork == null) {
+ fetchWork = new FetchWork(partDir, partDesc);
+ }
+ // set alias to fetch work
+ newLocalWork.getAliasToFetchWork().put(alias, fetchWork);
+ }
+ // remove small table ailias from aliasToWork;Avoid concurrent modification
+ for (String alias : smallTableAliasList) {
+ newWork.getAliasToWork().remove(alias);
+ }
+
+ // set up local work
+ newWork.setMapLocalWork(newLocalWork);
+ // remove reducer
+ newWork.setReducer(null);
+ // return the big table alias
+ if (bigTableAlias == null) {
+ throw new SemanticException("Big Table Alias is null");
+ }
+ return bigTableAlias;
+ }
+
+ public static String genMapJoinOpAndLocalWork(MapredWork newWork, JoinOperator op, int mapJoinPos)
+ throws SemanticException {
+ try {
+ LinkedHashMap, OpParseContext> opParseCtxMap = newWork
+ .getOpParseCtxMap();
+ QBJoinTree newJoinTree = newWork.getJoinTree();
+ // generate the map join operator; already checked the map join
+ MapJoinOperator newMapJoinOp = MapJoinProcessor.convertMapJoin(opParseCtxMap, op,
+ newJoinTree, mapJoinPos, true);
+ // generate the local work and return the big table alias
+ String bigTableAlias = MapJoinProcessor
+ .genMapJoinLocalWork(newWork, newMapJoinOp, mapJoinPos);
+ // clean up the mapred work
+ newWork.setOpParseCtxMap(null);
+ newWork.setJoinTree(null);
+
+ return bigTableAlias;
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new SemanticException("Generate New MapJoin Opertor Exeception " + e.getMessage());
+ }
+
+ }
+
+ /**
* convert a regular join to a a map-side join.
*
* @param op
@@ -108,29 +260,27 @@
* @param qbJoin
* qb join tree
* @param mapJoinPos
- * position of the source to be read as part of map-reduce framework.
- * All other sources are cached in memory
+ * position of the source to be read as part of map-reduce framework. All other sources
+ * are cached in memory
*/
- private MapJoinOperator convertMapJoin(ParseContext pctx, JoinOperator op,
- QBJoinTree joinTree, int mapJoinPos) throws SemanticException {
+ public static MapJoinOperator convertMapJoin(
+ LinkedHashMap, OpParseContext> opParseCtxMap,
+ JoinOperator op, QBJoinTree joinTree, int mapJoinPos, boolean noCheckOuterJoin)
+ throws SemanticException {
// outer join cannot be performed on a table which is being cached
JoinDesc desc = op.getConf();
- org.apache.hadoop.hive.ql.plan.JoinCondDesc[] condns = desc.getConds();
- HiveConf hiveConf = pGraphContext.getConf();
- boolean noCheckOuterJoin = HiveConf.getBoolVar(hiveConf,
- HiveConf.ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)
- && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN);
+ JoinCondDesc[] condns = desc.getConds();
+ Byte[] tagOrder = desc.getTagOrder();
+
if (!noCheckOuterJoin) {
checkMapJoin(mapJoinPos, condns);
}
- RowResolver oldOutputRS = pctx.getOpParseCtx().get(op).getRR();
+ RowResolver oldOutputRS = opParseCtxMap.get(op).getRowResolver();
RowResolver outputRS = new RowResolver();
ArrayList outputColumnNames = new ArrayList();
Map> keyExprMap = new HashMap>();
Map> valueExprMap = new HashMap>();
- HashMap> filterMap =
- new HashMap>();
// Walk over all the sources (which are guaranteed to be reduce sink
// operators).
@@ -141,13 +291,14 @@
List> newParentOps = new ArrayList>();
List> oldReduceSinkParentOps = new ArrayList>();
Map colExprMap = new HashMap();
+ HashMap> columnTransfer = new HashMap>();
+
// found a source which is not to be stored in memory
if (leftSrc != null) {
// assert mapJoinPos == 0;
Operator extends Serializable> parentOp = parentOps.get(0);
assert parentOp.getParentOperators().size() == 1;
- Operator extends Serializable> grandParentOp = parentOp
- .getParentOperators().get(0);
+ Operator extends Serializable> grandParentOp = parentOp.getParentOperators().get(0);
oldReduceSinkParentOps.add(parentOp);
grandParentOp.removeChild(parentOp);
newParentOps.add(grandParentOp);
@@ -159,8 +310,7 @@
if (src != null) {
Operator extends Serializable> parentOp = parentOps.get(pos);
assert parentOp.getParentOperators().size() == 1;
- Operator extends Serializable> grandParentOp = parentOp
- .getParentOperators().get(0);
+ Operator extends Serializable> grandParentOp = parentOp.getParentOperators().get(0);
grandParentOp.removeChild(parentOp);
oldReduceSinkParentOps.add(parentOp);
@@ -171,21 +321,21 @@
// get the join keys from old parent ReduceSink operators
for (pos = 0; pos < newParentOps.size(); pos++) {
- ReduceSinkOperator oldPar = (ReduceSinkOperator) oldReduceSinkParentOps
- .get(pos);
+ ReduceSinkOperator oldPar = (ReduceSinkOperator) oldReduceSinkParentOps.get(pos);
ReduceSinkDesc rsconf = oldPar.getConf();
Byte tag = (byte) rsconf.getTag();
List keys = rsconf.getKeyCols();
keyExprMap.put(tag, keys);
+
+ // set column transfer
+ HashMap map = (HashMap) oldPar.getColumnExprMap();
+ columnTransfer.put(tag, map);
}
// create the map-join operator
for (pos = 0; pos < newParentOps.size(); pos++) {
- RowResolver inputRS = pGraphContext.getOpParseCtx().get(
- newParentOps.get(pos)).getRR();
-
+ RowResolver inputRS = opParseCtxMap.get(newParentOps.get(pos)).getRowResolver();
List values = new ArrayList();
- List filterDesc = new ArrayList();
Iterator keysIter = inputRS.getTableNames().iterator();
while (keysIter.hasNext()) {
@@ -202,35 +352,48 @@
String outputCol = oldValueInfo.getInternalName();
if (outputRS.get(key, field) == null) {
outputColumnNames.add(outputCol);
- ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(),
- valueInfo.getInternalName(), valueInfo.getTabAlias(), valueInfo
- .getIsVirtualCol());
+ ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(), valueInfo
+ .getInternalName(), valueInfo.getTabAlias(), valueInfo.getIsVirtualCol());
values.add(colDesc);
- outputRS.put(key, field, new ColumnInfo(outputCol, valueInfo
- .getType(), valueInfo.getTabAlias(), valueInfo
- .getIsVirtualCol(),valueInfo.isHiddenVirtualCol()));
+ outputRS.put(key, field, new ColumnInfo(outputCol, valueInfo.getType(), valueInfo
+ .getTabAlias(), valueInfo.getIsVirtualCol(), valueInfo.isHiddenVirtualCol()));
colExprMap.put(outputCol, colDesc);
}
}
}
- TypeCheckCtx tcCtx = new TypeCheckCtx(inputRS);
- for (ASTNode cond : joinTree.getFilters().get((byte)pos)) {
+ valueExprMap.put(new Byte((byte) pos), values);
+ }
- ExprNodeDesc filter =
- (ExprNodeDesc)TypeCheckProcFactory.genExprNode(cond, tcCtx).get(cond);
- if (filter == null) {
- throw new SemanticException(tcCtx.getError());
+ Map> filterMap = desc.getFilters();
+ for (Map.Entry> entry : filterMap.entrySet()) {
+ Byte srcAlias = entry.getKey();
+ List columnDescList = entry.getValue();
+
+ for (ExprNodeDesc nodeExpr : columnDescList) {
+ ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) nodeExpr;
+ for (ExprNodeDesc childDesc : funcDesc.getChildExprs()) {
+ if (!(childDesc instanceof ExprNodeColumnDesc)) {
+ continue;
+ }
+ ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) childDesc;
+ // reset columns
+ String column = columnDesc.getColumn();
+ String newColumn = null;
+ HashMap map = columnTransfer.get(srcAlias);
+ ExprNodeColumnDesc tmpDesc = (ExprNodeColumnDesc) map.get(column);
+ if (tmpDesc != null) {
+ newColumn = tmpDesc.getColumn();
+ }
+ if (newColumn == null) {
+ throw new SemanticException("No Column name found in parent reduce sink op");
+ }
+ columnDesc.setColumn(newColumn);
}
- filterDesc.add(filter);
}
-
- valueExprMap.put(new Byte((byte) pos), values);
- filterMap.put(new Byte((byte) pos), filterDesc);
}
- org.apache.hadoop.hive.ql.plan.JoinCondDesc[] joinCondns = op.getConf()
- .getConds();
+ JoinCondDesc[] joinCondns = op.getConf().getConds();
Operator[] newPar = new Operator[newParentOps.size()];
pos = 0;
@@ -248,9 +411,25 @@
.getFieldSchemasFromColumnList(keyCols, "mapjoinkey"));
List valueTableDescs = new ArrayList();
+ List valueFiltedTableDescs = new ArrayList();
for (pos = 0; pos < newParentOps.size(); pos++) {
List valueCols = valueExprMap.get(new Byte((byte) pos));
+ int length = valueCols.size();
+ List valueFilteredCols = new ArrayList(length);
+ // deep copy expr node desc
+ for (int i = 0; i < length; i++) {
+ valueFilteredCols.add(valueCols.get(i).clone());
+ }
+ List valueFilters = filterMap.get(new Byte((byte) pos));
+
+ if (valueFilters != null && valueFilters.size() != 0 && pos != mapJoinPos) {
+ ExprNodeColumnDesc isFilterDesc = new ExprNodeColumnDesc(TypeInfoFactory
+ .getPrimitiveTypeInfo(Constants.BOOLEAN_TYPE_NAME), "filter", "filter", false);
+ valueFilteredCols.add(isFilterDesc);
+ }
+
+
keyOrder = new StringBuilder();
for (int i = 0; i < valueCols.size(); i++) {
keyOrder.append("+");
@@ -258,16 +437,23 @@
TableDesc valueTableDesc = PlanUtils.getMapJoinValueTableDesc(PlanUtils
.getFieldSchemasFromColumnList(valueCols, "mapjoinvalue"));
+ TableDesc valueFilteredTableDesc = PlanUtils.getMapJoinValueTableDesc(PlanUtils
+ .getFieldSchemasFromColumnList(valueFilteredCols, "mapjoinvalue"));
valueTableDescs.add(valueTableDesc);
+ valueFiltedTableDescs.add(valueFilteredTableDesc);
}
+ MapJoinDesc mapJoinDescriptor = new MapJoinDesc(keyExprMap, keyTableDesc, valueExprMap,
+ valueTableDescs, valueFiltedTableDescs, outputColumnNames, mapJoinPos, joinCondns,
+ filterMap, op.getConf().getNoOuterJoin());
+ mapJoinDescriptor.setTagOrder(tagOrder);
- MapJoinOperator mapJoinOp = (MapJoinOperator) putOpInsertMap(
- OperatorFactory.getAndMakeChild(new MapJoinDesc(keyExprMap,
- keyTableDesc, valueExprMap, valueTableDescs, outputColumnNames,
- mapJoinPos, joinCondns, filterMap, op.getConf().getNoOuterJoin()),
- new RowSchema(outputRS.getColumnInfos()), newPar), outputRS);
+ MapJoinOperator mapJoinOp = (MapJoinOperator) OperatorFactory.getAndMakeChild(
+ mapJoinDescriptor, new RowSchema(outputRS.getColumnInfos()), newPar);
+ OpParseContext ctx = new OpParseContext(outputRS);
+ opParseCtxMap.put(mapJoinOp, ctx);
+
mapJoinOp.getConf().setReversedExprs(op.getConf().getReversedExprs());
mapJoinOp.setColumnExprMap(colExprMap);
@@ -283,37 +469,59 @@
op.setChildOperators(null);
op.setParentOperators(null);
+ return mapJoinOp;
+ }
+
+ public MapJoinOperator generateMapJoinOperator(ParseContext pctx, JoinOperator op,
+ QBJoinTree joinTree, int mapJoinPos) throws SemanticException {
+ HiveConf hiveConf = pctx.getConf();
+ boolean noCheckOuterJoin = HiveConf.getBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)
+ && HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN);
+
+
+ LinkedHashMap, OpParseContext> opParseCtxMap = pctx
+ .getOpParseCtx();
+ MapJoinOperator mapJoinOp = convertMapJoin(opParseCtxMap, op, joinTree, mapJoinPos,
+ noCheckOuterJoin);
// create a dummy select to select all columns
genSelectPlan(pctx, mapJoinOp);
return mapJoinOp;
}
- public static void checkMapJoin(int mapJoinPos,
- org.apache.hadoop.hive.ql.plan.JoinCondDesc[] condns)
- throws SemanticException {
- for (org.apache.hadoop.hive.ql.plan.JoinCondDesc condn : condns) {
- if (condn.getType() == JoinDesc.FULL_OUTER_JOIN) {
- throw new SemanticException(ErrorMsg.NO_OUTER_MAPJOIN.getMsg());
+ public static HashSet getSmallTableOnlySet(JoinCondDesc[] condns) {
+ HashSet smallTableOnlySet = new HashSet();
+
+ for (JoinCondDesc condn : condns) {
+ int joinType = condn.getType();
+ if (joinType == JoinDesc.FULL_OUTER_JOIN) {
+ return null;
+ } else if (joinType == JoinDesc.LEFT_OUTER_JOIN || joinType == JoinDesc.LEFT_SEMI_JOIN) {
+ smallTableOnlySet.add(condn.getRight());
+ } else if (joinType == JoinDesc.RIGHT_OUTER_JOIN) {
+ smallTableOnlySet.add(condn.getLeft());
}
- if ((condn.getType() == JoinDesc.LEFT_OUTER_JOIN)
- && (condn.getLeft() != mapJoinPos)) {
- throw new SemanticException(ErrorMsg.NO_OUTER_MAPJOIN.getMsg());
- }
- if ((condn.getType() == JoinDesc.RIGHT_OUTER_JOIN)
- && (condn.getRight() != mapJoinPos)) {
- throw new SemanticException(ErrorMsg.NO_OUTER_MAPJOIN.getMsg());
- }
}
+
+ return smallTableOnlySet;
}
- private void genSelectPlan(ParseContext pctx, MapJoinOperator input)
- throws SemanticException {
+ public static void checkMapJoin(int mapJoinPos, JoinCondDesc[] condns) throws SemanticException {
+ HashSet smallTableOnlySet = MapJoinProcessor.getSmallTableOnlySet(condns);
+
+ if (smallTableOnlySet == null || smallTableOnlySet.contains(mapJoinPos)) {
+ throw new SemanticException(ErrorMsg.NO_OUTER_MAPJOIN.getMsg());
+ }
+ return;
+ }
+
+ private void genSelectPlan(ParseContext pctx, MapJoinOperator input) throws SemanticException {
List> childOps = input.getChildOperators();
input.setChildOperators(null);
// create a dummy select - This select is needed by the walker to split the
// mapJoin later on
- RowResolver inputRR = pctx.getOpParseCtx().get(input).getRR();
+ RowResolver inputRR = pctx.getOpParseCtx().get(input).getRowResolver();
ArrayList exprs = new ArrayList();
ArrayList outputs = new ArrayList();
@@ -326,20 +534,19 @@
String internalName = outputCols.get(i);
String[] nm = inputRR.reverseLookup(internalName);
ColumnInfo valueInfo = inputRR.get(nm[0], nm[1]);
- ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(),
- valueInfo.getInternalName(), nm[0], valueInfo.getIsVirtualCol());
+ ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(), valueInfo
+ .getInternalName(), nm[0], valueInfo.getIsVirtualCol());
exprs.add(colDesc);
outputs.add(internalName);
- outputRS.put(nm[0], nm[1], new ColumnInfo(internalName, valueInfo
- .getType(), nm[0], valueInfo.getIsVirtualCol(), valueInfo.isHiddenVirtualCol()));
+ outputRS.put(nm[0], nm[1], new ColumnInfo(internalName, valueInfo.getType(), nm[0], valueInfo
+ .getIsVirtualCol(), valueInfo.isHiddenVirtualCol()));
colExprMap.put(internalName, colDesc);
}
SelectDesc select = new SelectDesc(exprs, outputs, false);
- SelectOperator sel = (SelectOperator) putOpInsertMap(
- OperatorFactory.getAndMakeChild(select, new RowSchema(inputRR
- .getColumnInfos()), input), inputRR);
+ SelectOperator sel = (SelectOperator) putOpInsertMap(OperatorFactory.getAndMakeChild(select,
+ new RowSchema(inputRR.getColumnInfos()), input), inputRR);
sel.setColumnExprMap(colExprMap);
@@ -357,11 +564,10 @@
* join operator
* @param qbJoin
* qb join tree
- * @return -1 if it cannot be converted to a map-side join, position of the
- * map join node otherwise
+ * @return -1 if it cannot be converted to a map-side join, position of the map join node
+ * otherwise
*/
- private int mapSideJoin(JoinOperator op, QBJoinTree joinTree)
- throws SemanticException {
+ private int mapSideJoin(JoinOperator op, QBJoinTree joinTree) throws SemanticException {
int mapJoinPos = -1;
if (joinTree.isMapSideJoin()) {
int pos = 0;
@@ -387,8 +593,8 @@
// support this by randomly
// leaving some table from the list of tables to be cached
if (mapJoinPos == -1) {
- throw new SemanticException(ErrorMsg.INVALID_MAPJOIN_HINT
- .getMsg(pGraphContext.getQB().getParseInfo().getHints()));
+ throw new SemanticException(ErrorMsg.INVALID_MAPJOIN_HINT.getMsg(pGraphContext.getQB()
+ .getParseInfo().getHints()));
}
}
@@ -396,8 +602,8 @@
}
/**
- * Transform the query tree. For each join, check if it is a map-side join
- * (user specified). If yes, convert it to a map-side join.
+ * Transform the query tree. For each join, check if it is a map-side join (user specified). If
+ * yes, convert it to a map-side join.
*
* @param pactx
* current parse context
@@ -410,22 +616,20 @@
if (pGraphContext.getJoinContext() != null) {
Map joinMap = new HashMap();
Map mapJoinMap = pGraphContext.getMapJoinContext();
- if(mapJoinMap == null) {
- mapJoinMap = new HashMap ();
+ if (mapJoinMap == null) {
+ mapJoinMap = new HashMap();
pGraphContext.setMapJoinContext(mapJoinMap);
}
- Set> joinCtx = pGraphContext
- .getJoinContext().entrySet();
- Iterator> joinCtxIter = joinCtx
- .iterator();
+ Set> joinCtx = pGraphContext.getJoinContext().entrySet();
+ Iterator> joinCtxIter = joinCtx.iterator();
while (joinCtxIter.hasNext()) {
Map.Entry joinEntry = joinCtxIter.next();
JoinOperator joinOp = joinEntry.getKey();
QBJoinTree qbJoin = joinEntry.getValue();
int mapJoinPos = mapSideJoin(joinOp, qbJoin);
if (mapJoinPos >= 0) {
- MapJoinOperator mapJoinOp = convertMapJoin(pactx, joinOp, qbJoin, mapJoinPos);
+ MapJoinOperator mapJoinOp = generateMapJoinOperator(pactx, joinOp, qbJoin, mapJoinPos);
listMapJoinOps.add(mapJoinOp);
mapJoinMap.put(mapJoinOp, qbJoin);
} else {
@@ -444,19 +648,15 @@
// the operator stack.
// The dispatcher generates the plan from the operator tree
Map opRules = new LinkedHashMap();
- opRules.put(new RuleRegExp(new String("R0"), "MAPJOIN%"),
- getCurrentMapJoin());
- opRules.put(new RuleRegExp(new String("R1"), "MAPJOIN%.*FS%"),
- getMapJoinFS());
- opRules.put(new RuleRegExp(new String("R2"), "MAPJOIN%.*RS%"),
- getMapJoinDefault());
- opRules.put(new RuleRegExp(new String("R4"), "MAPJOIN%.*UNION%"),
- getMapJoinDefault());
+ opRules.put(new RuleRegExp(new String("R0"), "MAPJOIN%"), getCurrentMapJoin());
+ opRules.put(new RuleRegExp(new String("R1"), "MAPJOIN%.*FS%"), getMapJoinFS());
+ opRules.put(new RuleRegExp(new String("R2"), "MAPJOIN%.*RS%"), getMapJoinDefault());
+ opRules.put(new RuleRegExp(new String("R4"), "MAPJOIN%.*UNION%"), getMapJoinDefault());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
- Dispatcher disp = new DefaultRuleDispatcher(getDefault(), opRules,
- new MapJoinWalkerCtx(listMapJoinOpsNoRed, pGraphContext));
+ Dispatcher disp = new DefaultRuleDispatcher(getDefault(), opRules, new MapJoinWalkerCtx(
+ listMapJoinOpsNoRed, pGraphContext));
GraphWalker ogw = new GenMapRedWalker(disp);
ArrayList topNodes = new ArrayList();
@@ -483,7 +683,7 @@
MapJoinWalkerCtx ctx = (MapJoinWalkerCtx) procCtx;
MapJoinOperator mapJoin = (MapJoinOperator) nd;
if (ctx.getListRejectedMapJoins() != null && !ctx.getListRejectedMapJoins().contains(mapJoin)) {
- //for rule: MapJoin%.*MapJoin
+ // for rule: MapJoin%.*MapJoin
// have a child mapjoin. if the the current mapjoin is on a local work,
// will put the current mapjoin in the rejected list.
Boolean bigBranch = findGrandChildSubqueryMapjoin(ctx, mapJoin);
@@ -491,7 +691,7 @@
ctx.setCurrMapJoinOp(mapJoin);
return null;
}
- if(bigBranch) {
+ if (bigBranch) {
addNoReducerMapJoinToCtx(ctx, mapJoin);
} else {
addRejectMapJoinToCtx(ctx, mapJoin);
@@ -505,28 +705,24 @@
private Boolean findGrandChildSubqueryMapjoin(MapJoinWalkerCtx ctx, MapJoinOperator mapJoin) {
Operator extends Serializable> parent = mapJoin;
while (true) {
- if(parent.getChildOperators() == null || parent.getChildOperators().size() != 1) {
+ if (parent.getChildOperators() == null || parent.getChildOperators().size() != 1) {
return null;
}
Operator extends Serializable> ch = parent.getChildOperators().get(0);
- if(ch instanceof MapJoinOperator) {
- if (!nonSubqueryMapJoin(ctx.getpGraphContext(), (MapJoinOperator) ch,
- mapJoin)) {
- if (ch.getParentOperators().indexOf(parent) == ((MapJoinOperator) ch)
- .getConf().getPosBigTable()) {
- //not come from the local branch
+ if (ch instanceof MapJoinOperator) {
+ if (!nonSubqueryMapJoin(ctx.getpGraphContext(), (MapJoinOperator) ch, mapJoin)) {
+ if (ch.getParentOperators().indexOf(parent) == ((MapJoinOperator) ch).getConf()
+ .getPosBigTable()) {
+ // not come from the local branch
return true;
}
}
return false; // not from a sub-query.
}
- if ((ch instanceof JoinOperator)
- || (ch instanceof UnionOperator)
- || (ch instanceof ReduceSinkOperator)
- || (ch instanceof LateralViewJoinOperator)
- || (ch instanceof GroupByOperator)
- || (ch instanceof ScriptOperator)) {
+ if ((ch instanceof JoinOperator) || (ch instanceof UnionOperator)
+ || (ch instanceof ReduceSinkOperator) || (ch instanceof LateralViewJoinOperator)
+ || (ch instanceof GroupByOperator) || (ch instanceof ScriptOperator)) {
return null;
}
@@ -534,11 +730,11 @@
}
}
- private boolean nonSubqueryMapJoin(ParseContext pGraphContext,
- MapJoinOperator mapJoin, MapJoinOperator parentMapJoin) {
+ private boolean nonSubqueryMapJoin(ParseContext pGraphContext, MapJoinOperator mapJoin,
+ MapJoinOperator parentMapJoin) {
QBJoinTree joinTree = pGraphContext.getMapJoinContext().get(mapJoin);
QBJoinTree parentJoinTree = pGraphContext.getMapJoinContext().get(parentMapJoin);
- if(joinTree.getJoinSrc() != null && joinTree.getJoinSrc().equals(parentJoinTree)) {
+ if (joinTree.getJoinSrc() != null && joinTree.getJoinSrc().equals(parentJoinTree)) {
return true;
}
return false;
@@ -547,11 +743,11 @@
private static void addNoReducerMapJoinToCtx(MapJoinWalkerCtx ctx,
AbstractMapJoinOperator extends MapJoinDesc> mapJoin) {
- if (ctx.getListRejectedMapJoins() != null
- && ctx.getListRejectedMapJoins().contains(mapJoin)) {
+ if (ctx.getListRejectedMapJoins() != null && ctx.getListRejectedMapJoins().contains(mapJoin)) {
return;
}
- List> listMapJoinsNoRed = ctx.getListMapJoinsNoRed();
+ List> listMapJoinsNoRed = ctx
+ .getListMapJoinsNoRed();
if (listMapJoinsNoRed == null) {
listMapJoinsNoRed = new ArrayList>();
}
@@ -565,10 +761,11 @@
AbstractMapJoinOperator extends MapJoinDesc> mapjoin) {
// current map join is null means it has been handled by CurrentMapJoin
// process.
- if(mapjoin == null) {
+ if (mapjoin == null) {
return;
}
- List> listRejectedMapJoins = ctx.getListRejectedMapJoins();
+ List> listRejectedMapJoins = ctx
+ .getListRejectedMapJoins();
if (listRejectedMapJoins == null) {
listRejectedMapJoins = new ArrayList>();
}
@@ -576,15 +773,15 @@
listRejectedMapJoins.add(mapjoin);
}
- if (ctx.getListMapJoinsNoRed() != null
- && ctx.getListMapJoinsNoRed().contains(mapjoin)) {
+ if (ctx.getListMapJoinsNoRed() != null && ctx.getListMapJoinsNoRed().contains(mapjoin)) {
ctx.getListMapJoinsNoRed().remove(mapjoin);
}
ctx.setListRejectedMapJoins(listRejectedMapJoins);
}
- private static int findGrandparentBranch(Operator extends Serializable> currOp, Operator extends Serializable> grandParent) {
+ private static int findGrandparentBranch(Operator extends Serializable> currOp,
+ Operator extends Serializable> grandParent) {
int pos = -1;
for (int i = 0; i < currOp.getParentOperators().size(); i++) {
List> parentOpList = new LinkedList>();
@@ -592,14 +789,14 @@
boolean found = false;
while (!parentOpList.isEmpty()) {
Operator extends Serializable> p = parentOpList.remove(0);
- if(p == grandParent) {
+ if (p == grandParent) {
found = true;
break;
- } else if (p.getParentOperators() != null){
+ } else if (p.getParentOperators() != null) {
parentOpList.addAll(p.getParentOperators());
}
}
- if(found) {
+ if (found) {
pos = i;
break;
}
@@ -626,8 +823,7 @@
.getListRejectedMapJoins();
// the mapjoin has already been handled
- if ((listRejectedMapJoins != null)
- && (listRejectedMapJoins.contains(mapJoin))) {
+ if ((listRejectedMapJoins != null) && (listRejectedMapJoins.contains(mapJoin))) {
return null;
}
addNoReducerMapJoinToCtx(ctx, mapJoin);
@@ -701,7 +897,8 @@
* @param listMapJoinsNoRed
* @param pGraphContext2
*/
- public MapJoinWalkerCtx(List> listMapJoinsNoRed, ParseContext pGraphContext) {
+ public MapJoinWalkerCtx(List> listMapJoinsNoRed,
+ ParseContext pGraphContext) {
this.listMapJoinsNoRed = listMapJoinsNoRed;
currMapJoinOp = null;
listRejectedMapJoins = new ArrayList>();
@@ -719,7 +916,8 @@
* @param listMapJoinsNoRed
* the listMapJoins to set
*/
- public void setListMapJoins(List> listMapJoinsNoRed) {
+ public void setListMapJoins(
+ List> listMapJoinsNoRed) {
this.listMapJoinsNoRed = listMapJoinsNoRed;
}
Index: ql/src/java/org/apache/hadoop/hive/ql/ppd/OpWalkerInfo.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/ppd/OpWalkerInfo.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/ppd/OpWalkerInfo.java (working copy)
@@ -56,7 +56,7 @@
}
public RowResolver getRowResolver(Node op) {
- return opToParseCtxMap.get(op).getRR();
+ return opToParseCtxMap.get(op).getRowResolver();
}
public OpParseContext put(Operator extends Serializable> key,
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (working copy)
@@ -42,17 +42,20 @@
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.util.ReflectionUtils;
-public class HashTableSinkOperator extends TerminalOperator
- implements Serializable {
+
+public class HashTableSinkOperator extends TerminalOperator implements
+ Serializable {
private static final long serialVersionUID = 1L;
- private static final Log LOG = LogFactory.getLog(HashTableSinkOperator.class
- .getName());
+ private static final Log LOG = LogFactory.getLog(HashTableSinkOperator.class.getName());
// from abstract map join operator
/**
@@ -68,10 +71,8 @@
*/
protected transient Map> joinKeysStandardObjectInspectors;
- protected transient int posBigTableTag = -1; // one of the tables that is not
- // in memory
- protected transient int posBigTableAlias = -1; // one of the tables that is
- // not in memory
+ protected transient int posBigTableTag = -1; // one of the tables that is not in memory
+ protected transient int posBigTableAlias = -1; // one of the tables that is not in memory
transient int mapJoinRowsKey; // rows for a given key
protected transient RowContainer> emptyList = null;
@@ -114,7 +115,10 @@
private long rowNumber = 0;
protected transient LogHelper console;
+ private long hashTableScale;
+ private boolean isAbort = false;
+
public static class HashTableSinkObjectCtx {
ObjectInspector standardOI;
SerDe serde;
@@ -125,8 +129,8 @@
* @param standardOI
* @param serde
*/
- public HashTableSinkObjectCtx(ObjectInspector standardOI, SerDe serde,
- TableDesc tblDesc, Configuration conf) {
+ public HashTableSinkObjectCtx(ObjectInspector standardOI, SerDe serde, TableDesc tblDesc,
+ Configuration conf) {
this.standardOI = standardOI;
this.serde = serde;
this.tblDesc = tblDesc;
@@ -157,26 +161,29 @@
}
+ private static final transient String[] FATAL_ERR_MSG = {
+ null, // counter value 0 means no error
+ "Mapside join size exceeds hive.mapjoin.maxsize. "
+ + "Please increase that or remove the mapjoin hint."};
private final int metadataKeyTag = -1;
transient int[] metadataValueTag;
transient int maxMapJoinSize;
+
public HashTableSinkOperator() {
- // super();
- console = new LogHelper(LOG, true);
}
public HashTableSinkOperator(MapJoinOperator mjop) {
this.conf = new HashTableSinkDesc(mjop.getConf());
- console = new LogHelper(LOG);
}
+
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
+ boolean isSilent = HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVESESSIONSILENT);
+ console = new LogHelper(LOG, isSilent);
+ maxMapJoinSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAXMAPJOINSIZE);
- maxMapJoinSize = HiveConf.getIntVar(hconf,
- HiveConf.ConfVars.HIVEMAXMAPJOINSIZE);
-
numMapRowsRead = 0;
firstRow = true;
@@ -187,8 +194,7 @@
posBigTableAlias = order[posBigTableTag];
- // initialize some variables, which used to be initialized in
- // CommonJoinOperator
+ // initialize some variables, which used to be initialized in CommonJoinOperator
numAliases = conf.getExprs().size();
this.hconf = hconf;
totalSz = 0;
@@ -197,28 +203,25 @@
// process join keys
joinKeys = new HashMap>();
- JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), order,
- posBigTableAlias);
- joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(
- joinKeys, inputObjInspectors, posBigTableAlias);
+ JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), order, posBigTableAlias);
+ joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys,
+ inputObjInspectors, posBigTableAlias);
joinKeysStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
joinKeysObjectInspectors, posBigTableAlias);
// process join values
joinValues = new HashMap>();
- JoinUtil.populateJoinKeyValue(joinValues, conf.getExprs(), order,
- posBigTableAlias);
- joinValuesObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(
- joinValues, inputObjInspectors, posBigTableAlias);
+ JoinUtil.populateJoinKeyValue(joinValues, conf.getExprs(), order, posBigTableAlias);
+ joinValuesObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinValues,
+ inputObjInspectors, posBigTableAlias);
joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
joinValuesObjectInspectors, posBigTableAlias);
// process join filters
joinFilters = new HashMap>();
- JoinUtil.populateJoinKeyValue(joinFilters, conf.getFilters(), order,
- posBigTableAlias);
- joinFilterObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(
- joinFilters, inputObjInspectors, posBigTableAlias);
+ JoinUtil.populateJoinKeyValue(joinFilters, conf.getFilters(), order, posBigTableAlias);
+ joinFilterObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinFilters,
+ inputObjInspectors, posBigTableAlias);
if (noOuterJoin) {
rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors;
@@ -231,8 +234,7 @@
ArrayList rcOIs = new ArrayList();
rcOIs.addAll(joinValuesObjectInspectors.get(alias));
// for each alias, add object inspector for boolean as the last element
- rcOIs
- .add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+ rcOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
rowContainerObjectInspectors.put(alias, rcOIs);
}
rowContainerStandardObjectInspectors = getStandardObjectInspectors(rowContainerObjectInspectors);
@@ -245,52 +247,62 @@
mapJoinTables = new HashMap>();
+ int hashTableThreshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD);
+ float hashTableLoadFactor = HiveConf.getFloatVar(hconf,
+ HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR);
+ float hashTableMaxMemoryUsage = HiveConf.getFloatVar(hconf,
+ HiveConf.ConfVars.HIVEHASHTABLEMAXMEMORYUSAGE);
+ hashTableScale = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVEHASHTABLESCALE);
+ if (hashTableScale <= 0) {
+ hashTableScale = 1;
+ }
+
// initialize the hash tables for other tables
for (Byte pos : order) {
if (pos == posBigTableTag) {
continue;
}
- HashMapWrapper hashTable = new HashMapWrapper();
+ HashMapWrapper hashTable = new HashMapWrapper(
+ hashTableThreshold, hashTableLoadFactor, hashTableMaxMemoryUsage);
+
mapJoinTables.put(pos, hashTable);
}
}
+
+
protected static HashMap> getStandardObjectInspectors(
Map> aliasToObjectInspectors) {
HashMap> result = new HashMap>();
- for (Entry> oiEntry : aliasToObjectInspectors
- .entrySet()) {
+ for (Entry> oiEntry : aliasToObjectInspectors.entrySet()) {
Byte alias = oiEntry.getKey();
List oiList = oiEntry.getValue();
- ArrayList fieldOIList = new ArrayList(
- oiList.size());
+ ArrayList fieldOIList = new ArrayList(oiList.size());
for (int i = 0; i < oiList.size(); i++) {
- fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList
- .get(i), ObjectInspectorCopyOption.WRITABLE));
+ fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList.get(i),
+ ObjectInspectorCopyOption.WRITABLE));
}
result.put(alias, fieldOIList);
}
return result;
+
}
- public void generateMapMetaData() throws Exception {
+ private void setKeyMetaData() throws SerDeException {
TableDesc keyTableDesc = conf.getKeyTblDesc();
- SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc
- .getDeserializerClass(), null);
+ SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
+ null);
keySerializer.initialize(null, keyTableDesc.getProperties());
MapJoinMetaData.clear();
- MapJoinMetaData.put(Integer.valueOf(metadataKeyTag),
- new HashTableSinkObjectCtx(ObjectInspectorUtils
- .getStandardObjectInspector(keySerializer.getObjectInspector(),
- ObjectInspectorCopyOption.WRITABLE), keySerializer,
- keyTableDesc, hconf));
+ MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
+ ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
+ ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));
}
/*
- * This operator only process small tables Read the key/value pairs Load them
- * into hashtable
+ * This operator only process small tables Read the key/value pairs Load them into hashtable
*/
@Override
public void processOp(Object row, int tag) throws HiveException {
@@ -298,21 +310,21 @@
try {
if (firstRow) {
// generate the map metadata
- generateMapMetaData();
+ setKeyMetaData();
firstRow = false;
}
alias = order[tag];
// alias = (byte)tag;
// compute keys and values as StandardObjects
- AbstractMapJoinKey keyMap = JoinUtil.computeMapJoinKeys(row, joinKeys
- .get(alias), joinKeysObjectInspectors.get(alias));
+ AbstractMapJoinKey keyMap = JoinUtil.computeMapJoinKeys(row, joinKeys.get(alias),
+ joinKeysObjectInspectors.get(alias));
- Object[] value = JoinUtil.computeMapJoinValues(row,
- joinValues.get(alias), joinValuesObjectInspectors.get(alias),
- joinFilters.get(alias), joinFilterObjectInspectors.get(alias),
- noOuterJoin);
+ Object[] value = JoinUtil.computeMapJoinValues(row, joinValues.get(alias),
+ joinValuesObjectInspectors.get(alias), joinFilters.get(alias), joinFilterObjectInspectors
+ .get(alias), noOuterJoin);
+
HashMapWrapper hashTable = mapJoinTables
.get((byte) tag);
@@ -326,24 +338,20 @@
if (metadataValueTag[tag] == -1) {
metadataValueTag[tag] = order[tag];
-
- TableDesc valueTableDesc = conf.getValueTblDescs().get(tag);
- SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc
- .getDeserializerClass(), null);
- valueSerDe.initialize(null, valueTableDesc.getProperties());
-
- MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]),
- new HashTableSinkObjectCtx(ObjectInspectorUtils
- .getStandardObjectInspector(valueSerDe.getObjectInspector(),
- ObjectInspectorCopyOption.WRITABLE), valueSerDe,
- valueTableDesc, hconf));
+ setValueMetaData(tag);
}
// Construct externalizable objects for key and value
if (needNewKey) {
- MapJoinObjectValue valueObj = new MapJoinObjectValue(
- metadataValueTag[tag], res);
+ MapJoinObjectValue valueObj = new MapJoinObjectValue(metadataValueTag[tag], res);
+
rowNumber++;
+ if (rowNumber > hashTableScale && rowNumber % hashTableScale == 0) {
+ isAbort = hashTable.isAbort(rowNumber, console);
+ if (isAbort) {
+ throw new HiveException("RunOutOfMeomoryUsage");
+ }
+ }
hashTable.put(keyMap, valueObj);
}
@@ -352,13 +360,34 @@
res.add(value);
}
- } catch (Exception e) {
- e.printStackTrace();
+
+ } catch (SerDeException e) {
throw new HiveException(e);
}
}
+ private void setValueMetaData(int tag) throws SerDeException {
+ TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(tag);
+ SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
+ null);
+
+ valueSerDe.initialize(null, valueTableDesc.getProperties());
+
+ List newFields = rowContainerStandardObjectInspectors.get((Byte) alias);
+ int length = newFields.size();
+ List newNames = new ArrayList(length);
+ for (int i = 0; i < length; i++) {
+ String tmp = new String("tmp_" + i);
+ newNames.add(tmp);
+ }
+ StandardStructObjectInspector standardOI = ObjectInspectorFactory
+ .getStandardStructObjectInspector(newNames, newFields);
+
+ MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]), new HashTableSinkObjectCtx(
+ standardOI, valueSerDe, valueTableDesc, hconf));
+ }
+
@Override
public void closeOp(boolean abort) throws HiveException {
try {
@@ -371,31 +400,24 @@
.entrySet()) {
// get the key and value
Byte tag = hashTables.getKey();
- HashMapWrapper hashTable = hashTables
- .getValue();
+ HashMapWrapper hashTable = hashTables.getValue();
// get current input file name
- String bigBucketFileName = this.getExecContext()
- .getCurrentBigBucketFile();
+ String bigBucketFileName = this.getExecContext().getCurrentBigBucketFile();
if (bigBucketFileName == null || bigBucketFileName.length() == 0) {
bigBucketFileName = "-";
}
// get the tmp URI path; it will be a hdfs path if not local mode
- String tmpURIPath = Utilities.generatePath(tmpURI, tag,
- bigBucketFileName);
- console.printInfo(Utilities.now()
- + "\tDump the hashtable into file: " + tmpURIPath);
+ String tmpURIPath = PathUtil.generatePath(tmpURI, tag, bigBucketFileName);
+ hashTable.isAbort(rowNumber, console);
+ console.printInfo(Utilities.now() + "\tDump the hashtable into file: " + tmpURIPath);
// get the hashtable file and path
Path path = new Path(tmpURIPath);
FileSystem fs = path.getFileSystem(hconf);
File file = new File(path.toUri().getPath());
fs.create(path);
-
fileLength = hashTable.flushMemoryCacheToPersistent(file);
-
- console.printInfo(Utilities.now() + "\t Processing rows: "
- + rowNumber + "\t key number:" + hashTable.size());
- console.printInfo("Upload 1 File to: " + tmpURIPath + " File size: "
+ console.printInfo(Utilities.now() + "\tUpload 1 File to: " + tmpURIPath + " File size: "
+ fileLength);
hashTable.close();
@@ -411,7 +433,7 @@
/**
* Implements the getName function for the Node Interface.
- *
+ *
* @return the name of the operator
*/
@Override
@@ -424,4 +446,6 @@
return OperatorType.HASHTABLESINK;
}
+
+
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (working copy)
@@ -34,6 +34,7 @@
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.util.StringUtils;
@@ -42,8 +43,7 @@
* Task implementation.
**/
-public abstract class Task implements Serializable,
- Node {
+public abstract class Task implements Serializable, Node {
private static final long serialVersionUID = 1L;
protected transient boolean started;
@@ -59,7 +59,10 @@
protected transient HashMap taskCounters;
protected transient DriverContext driverContext;
protected transient boolean clonedConf = false;
+ protected Task extends Serializable> backupTask;
+ protected List> backupChildrenTasks = new ArrayList>();
+
// Descendants tasks who subscribe feeds from this task
protected transient List> feedSubscribers;
@@ -81,8 +84,7 @@
this.taskCounters = new HashMap();
}
- public void initialize(HiveConf conf, QueryPlan queryPlan,
- DriverContext driverContext) {
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) {
this.queryPlan = queryPlan;
isdone = false;
started = false;
@@ -103,8 +105,8 @@
}
/**
- * This method is called in the Driver on every task. It updates counters and
- * calls execute(), which is overridden in each task
+ * This method is called in the Driver on every task. It updates counters and calls execute(),
+ * which is overridden in each task
*
* @return return value of execute()
*/
@@ -127,8 +129,7 @@
}
/**
- * This method is overridden in each Task. TODO execute should return a
- * TaskHandle.
+ * This method is overridden in each Task. TODO execute should return a TaskHandle.
*
* @return status of executing the task
*/
@@ -160,10 +161,61 @@
return parentTasks;
}
+ public Task extends Serializable> getBackupTask() {
+ return backupTask;
+ }
+
+
+ public void setBackupTask(Task extends Serializable> backupTask) {
+ this.backupTask = backupTask;
+ }
+
+ public List> getBackupChildrenTasks() {
+ return backupChildrenTasks;
+ }
+
+ public void setBackupChildrenTasks(List> backupChildrenTasks) {
+ this.backupChildrenTasks = backupChildrenTasks;
+ }
+
+ public Task extends Serializable> getAndInitBackupTask(){
+ if (backupTask != null) {
+ //first set back the backup task with its children task.
+ for(Task extends Serializable> backupChild : backupChildrenTasks) {
+ backupChild.getParentTasks().add(backupTask);
+ }
+
+ //recursively remove task from its children tasks if this task doesn't have any parent task
+ this.removeFromChildrenTasks();
+ }
+ return backupTask;
+ }
+
+ public void removeFromChildrenTasks(){
+
+ List> childrenTasks = this.getChildTasks();
+ if (childrenTasks == null ) {
+ return;
+ }
+
+ for(Task extends Serializable> childTsk : childrenTasks) {
+ // remove this task from its children tasks
+ childTsk.getParentTasks().remove(this);
+
+ //recursively remove non-parent task from its children
+ List> siblingTasks = childTsk.getParentTasks();
+ if(siblingTasks == null || siblingTasks.size() == 0){
+ childTsk.removeFromChildrenTasks();
+ }
+ }
+
+ return ;
+ }
+
+
/**
- * The default dependent tasks are just child tasks, but different types
- * could implement their own (e.g. ConditionalTask will use the listTasks
- * as dependents).
+ * The default dependent tasks are just child tasks, but different types could implement their own
+ * (e.g. ConditionalTask will use the listTasks as dependents).
*
* @return a list of tasks that are dependent on this task.
*/
@@ -172,8 +224,8 @@
}
/**
- * Add a dependent task on the current task. Return if the dependency already
- * existed or is this a new one
+ * Add a dependent task on the current task. Return if the dependency already existed or is this a
+ * new one
*
* @return true if the task got added false if it already existed
*/
@@ -204,8 +256,7 @@
public void removeDependentTask(Task extends Serializable> dependent) {
if ((getChildTasks() != null) && (getChildTasks().contains(dependent))) {
getChildTasks().remove(dependent);
- if ((dependent.getParentTasks() != null)
- && (dependent.getParentTasks().contains(this))) {
+ if ((dependent.getParentTasks() != null) && (dependent.getParentTasks().contains(this))) {
dependent.getParentTasks().remove(this);
}
}
@@ -279,6 +330,10 @@
return false;
}
+ public boolean isMapRedLocalTask() {
+ return false;
+ }
+
public boolean hasReduce() {
return false;
}
@@ -288,8 +343,7 @@
}
/**
- * Should be overridden to return the type of the specific task among the
- * types in TaskType.
+ * Should be overridden to return the type of the specific task among the types in TaskType.
*
* @return TaskTypeType.* or -1 if not overridden
*/
@@ -299,21 +353,23 @@
}
/**
- * If this task uses any map-reduce intermediate data (either for reading
- * or for writing), localize them (using the supplied Context). Map-Reduce
- * intermediate directories are allocated using Context.getMRTmpFileURI()
- * and can be localized using localizeMRTmpFileURI().
+ * If this task uses any map-reduce intermediate data (either for reading or for writing),
+ * localize them (using the supplied Context). Map-Reduce intermediate directories are allocated
+ * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI().
*
- * This method is declared abstract to force any task code to explicitly
- * deal with this aspect of execution.
+ * This method is declared abstract to force any task code to explicitly deal with this aspect of
+ * execution.
*
- * @param ctx context object with which to localize
+ * @param ctx
+ * context object with which to localize
*/
abstract protected void localizeMRTmpFilesImpl(Context ctx);
/**
* Localize a task tree
- * @param ctx context object with which to localize
+ *
+ * @param ctx
+ * context object with which to localize
*/
public final void localizeMRTmpFiles(Context ctx) {
localizeMRTmpFilesImpl(ctx);
@@ -322,7 +378,7 @@
return;
}
- for (Task extends Serializable> t: childTasks) {
+ for (Task extends Serializable> t : childTasks) {
t.localizeMRTmpFiles(ctx);
}
}
@@ -330,12 +386,13 @@
/**
* Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its ancestor.
* Feed is a generic form of execution-time feedback (type, value) pair from one task to another
- * task. Examples include dynamic partitions (which are only available at execution time).
- * The MoveTask may pass the list of dynamic partitions to the StatsTask since after the
- * MoveTask the list of dynamic partitions are lost (MoveTask moves them to the table's
- * destination directory which is mixed with old partitions).
+ * task. Examples include dynamic partitions (which are only available at execution time). The
+ * MoveTask may pass the list of dynamic partitions to the StatsTask since after the MoveTask the
+ * list of dynamic partitions are lost (MoveTask moves them to the table's destination directory
+ * which is mixed with old partitions).
*
- * @param publisher this feed provider.
+ * @param publisher
+ * this feed provider.
*/
public void subscribeFeed(Task extends Serializable> publisher) {
if (publisher != this && publisher.ancestorOrSelf(this)) {
@@ -353,7 +410,7 @@
}
List> deps = getDependentTasks();
if (deps != null) {
- for (Task extends Serializable> d: deps) {
+ for (Task extends Serializable> d : deps) {
if (d.ancestorOrSelf(desc)) {
return true;
}
@@ -373,7 +430,7 @@
// push the feed to its subscribers
protected void pushFeed(FeedType feedType, Object feedValue) {
if (feedSubscribers != null) {
- for (Task extends Serializable> s: feedSubscribers) {
+ for (Task extends Serializable> s : feedSubscribers) {
s.receiveFeed(feedType, feedValue);
}
}
@@ -383,10 +440,38 @@
protected void receiveFeed(FeedType feedType, Object feedValue) {
}
- protected void cloneConf () {
+ protected void cloneConf() {
if (!clonedConf) {
clonedConf = true;
conf = new HiveConf(conf);
}
}
-}
\ No newline at end of file
+
+ public void replaceWithConditionalTask(ConditionalTask cndTsk, PhysicalContext physicalContext) {
+ // add this task into task tree
+ // set all parent tasks
+ List> parentTasks = this.getParentTasks();
+ this.setParentTasks(null);
+ if (parentTasks != null) {
+ for (Task extends Serializable> tsk : parentTasks) {
+ // make new generated task depends on all the parent tasks of current task.
+ tsk.addDependentTask(cndTsk);
+ // remove the current task from its original parent task's dependent task
+ tsk.removeDependentTask(this);
+ }
+ } else {
+ // remove from current root task and add conditional task to root tasks
+ physicalContext.removeFromRootTask(this);
+ physicalContext.addToRootTask(cndTsk);
+ }
+ // set all child tasks
+ List> oldChildTasks = this.getChildTasks();
+ if (oldChildTasks != null) {
+ for (Task extends Serializable> tsk : cndTsk.getListTasks()) {
+ for (Task extends Serializable> oldChild : oldChildTasks) {
+ tsk.addDependentTask(oldChild);
+ }
+ }
+ }
+ }
+}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (working copy)
@@ -49,7 +49,7 @@
private boolean isVirtualCol;
private transient TypeInfo type;
-
+
private boolean isHiddenVirtualCol;
public ColumnInfo() {
@@ -66,7 +66,7 @@
.getPrimitiveTypeInfoFromPrimitiveWritable(type), tabAlias,
isVirtualCol, false);
}
-
+
public ColumnInfo(String internalName, TypeInfo type, String tabAlias,
boolean isVirtualCol, boolean isHiddenVirtualCol) {
this.internalName = internalName;
@@ -99,7 +99,7 @@
public boolean getIsVirtualCol() {
return isVirtualCol;
}
-
+
public boolean isHiddenVirtualCol() {
return isHiddenVirtualCol;
}
@@ -120,4 +120,17 @@
return alias;
}
+ public void setTabAlias(String tabAlias) {
+ this.tabAlias = tabAlias;
+ }
+
+ public void setVirtualCol(boolean isVirtualCol) {
+ this.isVirtualCol = isVirtualCol;
+ }
+
+ public void setHiddenVirtualCol(boolean isHiddenVirtualCol) {
+ this.isHiddenVirtualCol = isHiddenVirtualCol;
+ }
+
+
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (working copy)
@@ -28,17 +28,17 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import java.util.Map.Entry;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.ExplainWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.Context;
/**
@@ -304,7 +304,7 @@
return;
}
dependeciesTaskSet.add(task);
-
+ boolean first = true;
out.print(indentString(indent));
out.printf("%s", task.getId());
if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) {
@@ -313,7 +313,7 @@
}
} else {
out.print(" depends on stages: ");
- boolean first = true;
+ first = true;
for (Task extends Serializable> parent : task.getParentTasks()) {
if (!first) {
out.print(", ");
@@ -321,22 +321,34 @@
first = false;
out.print(parent.getId());
}
+ }
- if (task instanceof ConditionalTask
- && ((ConditionalTask) task).getListTasks() != null) {
- out.print(" , consists of ");
- first = true;
- for (Task extends Serializable> con : ((ConditionalTask) task)
- .getListTasks()) {
- if (!first) {
- out.print(", ");
- }
- first = false;
- out.print(con.getId());
+ Task extends Serializable> cuurBackupTask = task.getBackupTask();
+ if(cuurBackupTask != null) {
+ out.print(" has a backup stage: ");
+ if (!first) {
+ out.print(", ");
+ }
+ first = false;
+ out.print(cuurBackupTask.getId());
+ }
+
+
+ if (task instanceof ConditionalTask
+ && ((ConditionalTask) task).getListTasks() != null) {
+ out.print(" , consists of ");
+ first = true;
+ for (Task extends Serializable> con : ((ConditionalTask) task)
+ .getListTasks()) {
+ if (!first) {
+ out.print(", ");
}
+ first = false;
+ out.print(con.getId());
}
+ }
- }
+
out.println();
if (task instanceof ConditionalTask
@@ -406,7 +418,7 @@
@Override
protected void localizeMRTmpFilesImpl(Context ctx) {
- // explain task has nothing to localize
+ // explain task has nothing to localize
// we don't expect to enter this code path at all
throw new RuntimeException ("Unexpected call");
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (working copy)
@@ -32,13 +32,13 @@
/**
* Conditional Task implementation.
*/
-public class ConditionalTask extends Task implements
- Serializable {
+public class ConditionalTask extends Task implements Serializable {
private static final long serialVersionUID = 1L;
private List> listTasks;
private boolean resolved = false;
+
private List> resTasks;
private ConditionalResolver resolver;
@@ -48,6 +48,7 @@
super();
}
+
@Override
public boolean isMapRedTask() {
for (Task extends Serializable> task : listTasks) {
@@ -71,8 +72,7 @@
}
@Override
- public void initialize(HiveConf conf, QueryPlan queryPlan,
- DriverContext driverContext) {
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) {
super.initialize(conf, queryPlan, driverContext);
}
@@ -80,30 +80,28 @@
public int execute(DriverContext driverContext) {
resTasks = resolver.getTasks(conf, resolverCtx);
resolved = true;
+
for (Task extends Serializable> tsk : getListTasks()) {
if (!resTasks.contains(tsk)) {
driverContext.getRunnable().remove(tsk);
- console.printInfo(ExecDriver.getJobEndMsg(""
- + Utilities.randGen.nextInt())
+ console.printInfo(ExecDriver.getJobEndMsg("" + Utilities.randGen.nextInt())
+ ", job is filtered out (removed at runtime).");
- if(tsk.isMapRedTask()) {
+ if (tsk.isMapRedTask()) {
driverContext.incCurJobNo(1);
}
- if (tsk.getChildTasks() != null) {
- for (Task extends Serializable> child : tsk.getChildTasks()) {
- child.parentTasks.remove(tsk);
- if (DriverContext.isLaunchable(child)) {
- driverContext.addToRunnable(child);
- }
- }
+ //recursively remove this task from its children's parent task
+ tsk.removeFromChildrenTasks();
+ } else {
+ // resolved task
+ if (!driverContext.getRunnable().contains(tsk)) {
+ driverContext.addToRunnable(tsk);
}
- } else if (!driverContext.getRunnable().contains(tsk)) {
- driverContext.addToRunnable(tsk);
}
}
return 0;
}
+
/**
* @return the resolver
*/
@@ -184,9 +182,8 @@
}
/**
- * Add a dependent task on the current conditional task. The task will not be
- * a direct child of conditional task. Actually it will be added as child task
- * of associated tasks.
+ * Add a dependent task on the current conditional task. The task will not be a direct child of
+ * conditional task. Actually it will be added as child task of associated tasks.
*
* @return true if the task got added false if it already existed
*/
@@ -204,7 +201,7 @@
@Override
protected void localizeMRTmpFilesImpl(Context ctx) {
if (getListTasks() != null) {
- for(Task extends Serializable> t: getListTasks()) {
+ for (Task extends Serializable> t : getListTasks()) {
t.localizeMRTmpFiles(ctx);
}
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (working copy)
@@ -89,7 +89,7 @@
// initialize map operator
mo.setChildren(job);
l4j.info(mo.dump(0));
- // initialize map local work
+ // initialize map local work
localWork = mrwork.getMapLocalWork();
execContext.setLocalWork(localWork);
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (working copy)
@@ -73,8 +73,9 @@
taskvec.add(new taskTuple(MapredLocalWork.class,
MapredLocalTask.class));
taskvec.add(new taskTuple(StatsWork.class,
- StatsTask.class));
+ StatsTask.class));
+
}
private static ThreadLocal tid = new ThreadLocal() {
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (working copy)
@@ -409,4 +409,7 @@
return null;
}
+
+
+
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (working copy)
@@ -33,6 +33,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -43,20 +44,20 @@
* the main memory hash table exceeds a certain threshold, new elements will go into the persistent
* hash table.
*/
+
public class HashMapWrapper implements Serializable {
+ private static final long serialVersionUID = 1L;
protected Log LOG = LogFactory.getLog(this.getClass().getName());
// default threshold for using main memory based HashMap
+
private static final int THRESHOLD = 1000000;
private static final float LOADFACTOR = 0.75f;
+ private static final float MEMORYUSAGE = 1;
- private double threshold; // threshold to put data into persistent hash table
- // instead
+ private float maxMemoryUsage;
private HashMap mHash; // main memory HashMap
-
-
-
protected transient LogHelper console;
private File dumpFile;
@@ -71,10 +72,9 @@
* @param threshold
* User specified threshold to store new values into persistent storage.
*/
- public HashMapWrapper(int threshold, float loadFactor) {
- this.threshold = 0.9;
+ public HashMapWrapper(int threshold, float loadFactor, float memoryUsage) {
+ maxMemoryUsage = memoryUsage;
mHash = new HashMap(threshold, loadFactor);
- console = new LogHelper(LOG);
memoryMXBean = ManagementFactory.getMemoryMXBean();
maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
LOG.info("maximum memory: " + maxMemory);
@@ -83,30 +83,28 @@
}
public HashMapWrapper(int threshold) {
- this(THRESHOLD, 0.75f);
+ this(threshold, LOADFACTOR, MEMORYUSAGE);
}
public HashMapWrapper() {
- this(THRESHOLD, LOADFACTOR);
+ this(THRESHOLD, LOADFACTOR, MEMORYUSAGE);
}
-
public V get(K key) {
return mHash.get(key);
}
-
public boolean put(K key, V value) throws HiveException {
// isAbort();
mHash.put(key, value);
return false;
}
+
public void remove(K key) {
mHash.remove(key);
}
-
/**
* Flush the main memory hash table into the persistent cache file
*
@@ -146,7 +144,6 @@
* @throws HiveException
*/
public void close() throws HiveException {
- // isAbort();
mHash.clear();
}
@@ -158,36 +155,25 @@
return mHash.size();
}
- private boolean isAbort() {
- int size = mHash.size();
- // if(size >= 1000000 && size % 1000000 == 0 ){
+ public boolean isAbort(long numRows,LogHelper console) {
System.gc();
System.gc();
+ int size = mHash.size();
long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
double rate = (double) usedMemory / (double) maxMemory;
long mem1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- console.printInfo("Hashtable size:\t" + size + "\tMemory usage:\t" + usedMemory + "\t rate:\t"
- + num.format(rate));
- return true;
-
+ console.printInfo(Utilities.now() + "\tProcessing rows:\t" + numRows + "\tHashtable size:\t"
+ + size + "\tMemory usage:\t" + usedMemory + "\trate:\t" + num.format(rate));
+ if (rate > (double) maxMemoryUsage) {
+ return true;
+ }
+ return false;
}
- public Log getLOG() {
- return LOG;
- }
-
public void setLOG(Log log) {
LOG = log;
}
- public double getThreshold() {
- return threshold;
- }
-
- public void setThreshold(double threshold) {
- this.threshold = threshold;
- }
-
public HashMap getMHash() {
return mHash;
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (working copy)
@@ -57,7 +57,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.ReflectionUtils;
-public class MapredLocalTask extends Task implements Serializable {
+public class MapredLocalTask extends Task implements Serializable {
private Map fetchOperators;
private JobConf job;
@@ -67,11 +67,11 @@
static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive"};
public static MemoryMXBean memoryMXBean;
- // not sure we need this exec context; but all the operators in the work
+ // not sure we need this exec context; but all the operators in the work
// will pass this context throught
private final ExecMapperContext execContext = new ExecMapperContext();
- public MapredLocalTask(){
+ public MapredLocalTask() {
super();
}
@@ -83,25 +83,23 @@
}
@Override
- public void initialize(HiveConf conf, QueryPlan queryPlan,
- DriverContext driverContext) {
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) {
super.initialize(conf, queryPlan, driverContext);
job = new JobConf(conf, ExecDriver.class);
}
- public static String now(){
+ public static String now() {
Calendar cal = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd hh:mm:ss");
- return sdf.format(cal.getTime());
+ return sdf.format(cal.getTime());
}
@Override
- public int execute(DriverContext driverContext){
- try{
- //generate the cmd line to run in the child jvm
- //String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN);
+ public int execute(DriverContext driverContext) {
+ try {
+ // generate the cmd line to run in the child jvm
Context ctx = driverContext.getCtx();
String hiveJar = conf.getJar();
@@ -115,16 +113,15 @@
LOG.info("Generating plan file " + planPath.toString());
Utilities.serializeMapRedLocalWork(plan, out);
- String isSilent = "true".equalsIgnoreCase(System
- .getProperty("test.silent")) ? "-nolog" : "";
+ String isSilent = "true".equalsIgnoreCase(System.getProperty("test.silent")) ? "-nolog" : "";
String jarCmd;
- jarCmd = hiveJar + " " + ExecDriver.class.getName() ;
+ jarCmd = hiveJar + " " + ExecDriver.class.getName();
String hiveConfArgs = ExecDriver.generateCmdLine(conf);
- String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan "
- + planPath.toString() + " " + isSilent + " " + hiveConfArgs;
+ String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + planPath.toString()
+ + " " + isSilent + " " + hiveConfArgs;
String workDir = (new File(".")).getCanonicalPath();
String files = ExecDriver.getResourceFiles(conf, SessionState.ResourceType.FILE);
@@ -134,16 +131,16 @@
workDir = (new Path(ctx.getLocalTmpFileURI())).toUri().getPath();
- if (! (new File(workDir)).mkdir()) {
- throw new IOException ("Cannot create tmp working dir: " + workDir);
+ if (!(new File(workDir)).mkdir()) {
+ throw new IOException("Cannot create tmp working dir: " + workDir);
}
- for (String f: StringUtils.split(files, ',')) {
+ for (String f : StringUtils.split(files, ',')) {
Path p = new Path(f);
String target = p.toUri().getPath();
String link = workDir + Path.SEPARATOR + p.getName();
if (FileUtil.symLink(target, link) != 0) {
- throw new IOException ("Cannot link to added file: " + target + " from: " + link);
+ throw new IOException("Cannot link to added file: " + target + " from: " + link);
}
}
}
@@ -166,31 +163,30 @@
Map variables = new HashMap(System.getenv());
// The user can specify the hadoop memory
- //if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) {
- // if we are running in local mode - then the amount of memory used
- // by the child jvm can no longer default to the memory used by the
- // parent jvm
- //int hadoopMem = conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);
- int hadoopMem= conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);;
+ // if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) {
+ // if we are running in local mode - then the amount of memory used
+ // by the child jvm can no longer default to the memory used by the
+ // parent jvm
+ // int hadoopMem = conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);
+ int hadoopMem = conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);
if (hadoopMem == 0) {
// remove env var that would default child jvm to use parent's memory
// as default. child jvm would use default memory for a hadoop client
variables.remove(HADOOP_MEM_KEY);
} else {
// user specified the memory for local mode hadoop run
- console.printInfo(" set heap size\t"+hadoopMem+"MB");
+ console.printInfo(" set heap size\t" + hadoopMem + "MB");
variables.put(HADOOP_MEM_KEY, String.valueOf(hadoopMem));
}
- //} else {
- // nothing to do - we are not running in local mode - only submitting
- // the job via a child process. in this case it's appropriate that the
- // child jvm use the same memory as the parent jvm
+ // } else {
+ // nothing to do - we are not running in local mode - only submitting
+ // the job via a child process. in this case it's appropriate that the
+ // child jvm use the same memory as the parent jvm
- //}
+ // }
if (variables.containsKey(HADOOP_OPTS_KEY)) {
- variables.put(HADOOP_OPTS_KEY, variables.get(HADOOP_OPTS_KEY)
- + hadoopOpts);
+ variables.put(HADOOP_OPTS_KEY, variables.get(HADOOP_OPTS_KEY) + hadoopOpts);
} else {
variables.put(HADOOP_OPTS_KEY, hadoopOpts);
}
@@ -205,10 +201,8 @@
// Run ExecDriver in another JVM
executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir));
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(),
- null, System.out);
- StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(),
- null, System.err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out);
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
outPrinter.start();
errPrinter.start();
@@ -217,10 +211,9 @@
if (exitVal != 0) {
LOG.error("Execution failed with exit status: " + exitVal);
- console.printError("Mapred Local Task Failed. Give up the map join stragery");
} else {
LOG.info("Execution completed successfully");
- console.printInfo("Mapred Local Task Running Successfully . Keep using map join stragery");
+ console.printInfo("Mapred Local Task Succeeded . Convert the Join into MapJoin");
}
return exitVal;
@@ -233,54 +226,59 @@
- public int executeFromChildJVM(DriverContext driverContext){
-
+ public int executeFromChildJVM(DriverContext driverContext) {
// check the local work
- if(work == null){
+ if (work == null) {
return -1;
}
memoryMXBean = ManagementFactory.getMemoryMXBean();
- console.printInfo(Utilities.now()+"\tStarting to luaunch local task to process map join ");
- console.printInfo("\tmaximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax());
+ long startTime = System.currentTimeMillis();
+ console.printInfo(Utilities.now()
+ + "\tStarting to luaunch local task to process map join;\tmaximum memory = "
+ + memoryMXBean.getHeapMemoryUsage().getMax());
fetchOperators = new HashMap();
Map fetchOpJobConfMap = new HashMap();
execContext.setJc(job);
- //set the local work, so all the operator can get this context
+ // set the local work, so all the operator can get this context
execContext.setLocalWork(work);
boolean inputFileChangeSenstive = work.getInputFileChangeSensitive();
- try{
+ try {
initializeOperators(fetchOpJobConfMap);
- //for each big table's bucket, call the start forward
- if(inputFileChangeSenstive){
- for( LinkedHashMap> bigTableBucketFiles:
- work.getBucketMapjoinContext().getAliasBucketFileNameMapping().values()){
- for(String bigTableBucket: bigTableBucketFiles.keySet()){
- startForward(inputFileChangeSenstive,bigTableBucket);
+ // for each big table's bucket, call the start forward
+ if (inputFileChangeSenstive) {
+ for (LinkedHashMap> bigTableBucketFiles : work
+ .getBucketMapjoinContext().getAliasBucketFileNameMapping().values()) {
+ for (String bigTableBucket : bigTableBucketFiles.keySet()) {
+ startForward(inputFileChangeSenstive, bigTableBucket);
}
}
- }else{
- startForward(inputFileChangeSenstive,null);
+ } else {
+ startForward(inputFileChangeSenstive, null);
}
- console.printInfo(now()+"\tEnd of local task ");
+ long currentTime = System.currentTimeMillis();
+ long elapsed = currentTime - startTime;
+ console.printInfo(Utilities.now() + "\tEnd of local task; Time Taken: "
+ + Utilities.showTime(elapsed) + " sec.");
} catch (Throwable e) {
- if (e instanceof OutOfMemoryError) {
+ if (e instanceof OutOfMemoryError
+ || (e instanceof HiveException && e.getMessage().equals("RunOutOfMeomoryUsage"))) {
// Don't create a new object if we are already out of memory
l4j.error("Out of Memory Error");
- console.printError("[Warning] Small table is too large to put into memory");
- return 2;
+ console
+ .printInfo("[Warning] HashTableSink Operator Failed. The memory usage is more than the configurated;");
+ return 3;
} else {
l4j.error("Hive Runtime Error: Map local work failed");
e.printStackTrace();
+ return 2;
}
- }finally{
- console.printInfo(Utilities.now()+"\tFinish running local task");
}
return 0;
}
private void startForward(boolean inputFileChangeSenstive, String bigTableBucket)
- throws Exception{
+ throws Exception {
for (Map.Entry entry : fetchOperators.entrySet()) {
int fetchOpRows = 0;
String alias = entry.getKey();
@@ -288,17 +286,17 @@
if (inputFileChangeSenstive) {
fetchOp.clearFetchContext();
- setUpFetchOpContext(fetchOp, alias,bigTableBucket);
+ setUpFetchOpContext(fetchOp, alias, bigTableBucket);
}
- //get the root operator
+ // get the root operator
Operator extends Serializable> forwardOp = work.getAliasToWork().get(alias);
- //walk through the operator tree
+ // walk through the operator tree
while (true) {
InspectableObject row = fetchOp.getNextRow();
if (row == null) {
if (inputFileChangeSenstive) {
- String fileName=this.getFileName(bigTableBucket);
+ String fileName = this.getFileName(bigTableBucket);
execContext.setCurrentBigBucketFile(fileName);
forwardOp.reset();
}
@@ -310,22 +308,23 @@
// check if any operator had a fatal error or early exit during
// execution
if (forwardOp.getDone()) {
- //ExecMapper.setDone(true);
+ // ExecMapper.setDone(true);
break;
}
}
}
}
+
private void initializeOperators(Map fetchOpJobConfMap)
- throws HiveException{
+ throws HiveException {
// this mapper operator is used to initialize all the operators
for (Map.Entry entry : work.getAliasToFetchWork().entrySet()) {
JobConf jobClone = new JobConf(job);
Operator extends Serializable> tableScan = work.getAliasToWork().get(entry.getKey());
boolean setColumnsNeeded = false;
- if(tableScan instanceof TableScanOperator) {
- ArrayList list = ((TableScanOperator)tableScan).getNeededColumnIDs();
+ if (tableScan instanceof TableScanOperator) {
+ ArrayList list = ((TableScanOperator) tableScan).getNeededColumnIDs();
if (list != null) {
ColumnProjectionUtils.appendReadColumnIDs(jobClone, list);
setColumnsNeeded = true;
@@ -336,18 +335,18 @@
ColumnProjectionUtils.setFullyReadColumns(jobClone);
}
- //create a fetch operator
- FetchOperator fetchOp = new FetchOperator(entry.getValue(),jobClone);
+ // create a fetch operator
+ FetchOperator fetchOp = new FetchOperator(entry.getValue(), jobClone);
fetchOpJobConfMap.put(fetchOp, jobClone);
fetchOperators.put(entry.getKey(), fetchOp);
l4j.info("fetchoperator for " + entry.getKey() + " created");
}
- //initilize all forward operator
+ // initilize all forward operator
for (Map.Entry entry : fetchOperators.entrySet()) {
- //get the forward op
+ // get the forward op
Operator extends Serializable> forwardOp = work.getAliasToWork().get(entry.getKey());
- //put the exe context into all the operators
+ // put the exe context into all the operators
forwardOp.setExecContext(execContext);
// All the operators need to be initialized before process
FetchOperator fetchOp = entry.getValue();
@@ -356,54 +355,58 @@
if (jobConf == null) {
jobConf = job;
}
- //initialize the forward operator
+ // initialize the forward operator
forwardOp.initialize(jobConf, new ObjectInspector[] {fetchOp.getOutputObjectInspector()});
l4j.info("fetchoperator for " + entry.getKey() + " initialized");
}
}
- private void setUpFetchOpContext(FetchOperator fetchOp, String alias,String currentInputFile)
- throws Exception {
+ private void setUpFetchOpContext(FetchOperator fetchOp, String alias, String currentInputFile)
+ throws Exception {
- BucketMapJoinContext bucketMatcherCxt = this.work
- .getBucketMapjoinContext();
+ BucketMapJoinContext bucketMatcherCxt = this.work.getBucketMapjoinContext();
- Class extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt
- .getBucketMatcherClass();
- BucketMatcher bucketMatcher = (BucketMatcher) ReflectionUtils.newInstance(
- bucketMatcherCls, null);
- bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
- .getAliasBucketFileNameMapping());
+ Class extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
+ BucketMatcher bucketMatcher = (BucketMatcher) ReflectionUtils.newInstance(bucketMatcherCls,
+ null);
+ bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt.getAliasBucketFileNameMapping());
- List aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile,
- bucketMatcherCxt.getMapJoinBigTableAlias(), alias);
+ List aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile, bucketMatcherCxt
+ .getMapJoinBigTableAlias(), alias);
Iterator iter = aliasFiles.iterator();
fetchOp.setupContext(iter, null);
}
- private String getFileName(String path){
- if(path== null || path.length()==0) {
+ private String getFileName(String path) {
+ if (path == null || path.length() == 0) {
return null;
}
- int last_separator = path.lastIndexOf(Path.SEPARATOR)+1;
+ int last_separator = path.lastIndexOf(Path.SEPARATOR) + 1;
String fileName = path.substring(last_separator);
return fileName;
}
+
@Override
- public void localizeMRTmpFilesImpl(Context ctx){
+ public void localizeMRTmpFilesImpl(Context ctx) {
}
@Override
+ public boolean isMapRedLocalTask() {
+ return true;
+ }
+
+ @Override
public String getName() {
return "MAPREDLOCAL";
}
+
@Override
public int getType() {
- //assert false;
+ // assert false;
return StageType.MAPREDLOCAL;
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (working copy)
@@ -194,12 +194,54 @@
* Returns an operator given the conf and a list of parent operators.
*/
public static Operator getAndMakeChild(T conf,
+ List> oplist) {
+ Operator ret = get((Class) conf.getClass());
+ ret.setConf(conf);
+ if (oplist.size() == 0) {
+ return (ret);
+ }
+
+ // Add the new operator as child of each of the passed in operators
+ for (Operator op : oplist) {
+ List children = op.getChildOperators();
+ if (children == null) {
+ children = new ArrayList();
+ }
+ children.add(ret);
+ op.setChildOperators(children);
+ }
+
+ // add parents for the newly created operator
+ List> parent = new ArrayList>();
+ for (Operator op : oplist) {
+ parent.add(op);
+ }
+
+ ret.setParentOperators(parent);
+
+ return (ret);
+ }
+
+ /**
+ * Returns an operator given the conf and a list of parent operators.
+ */
+ public static Operator getAndMakeChild(T conf,
RowSchema rwsch, Operator... oplist) {
Operator ret = getAndMakeChild(conf, oplist);
ret.setSchema(rwsch);
return (ret);
}
+ /**
+ * Returns an operator given the conf and a list of parent operators.
+ */
+ public static Operator getAndMakeChild(T conf,
+ RowSchema rwsch, List> oplist) {
+ Operator ret = getAndMakeChild(conf, oplist);
+ ret.setSchema(rwsch);
+ return (ret);
+ }
+
private OperatorFactory() {
// prevent instantiation
}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/PathUtil.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/PathUtil.java (revision 0)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/PathUtil.java (revision 0)
@@ -0,0 +1,20 @@
+package org.apache.hadoop.hive.ql.exec;
+
+import org.apache.hadoop.fs.Path;
+
+public class PathUtil {
+ public static String suffix=".hashtable";
+ public static String generatePath(String baseURI,Byte tag,String bigBucketFileName){
+ String path = new String(baseURI+Path.SEPARATOR+"-"+tag+"-"+bigBucketFileName+suffix);
+ return path;
+ }
+ public static String generateFileName(Byte tag,String bigBucketFileName){
+ String fileName = new String("-"+tag+"-"+bigBucketFileName+suffix);
+ return fileName;
+ }
+
+ public static String generateTmpURI(String baseURI,String id){
+ String tmpFileURI = new String(baseURI+Path.SEPARATOR+"HashTable-"+id);
+ return tmpFileURI;
+ }
+}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 1034599)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy)
@@ -104,6 +104,7 @@
protected transient int mapProgress = 0;
protected transient int reduceProgress = 0;
public static MemoryMXBean memoryMXBean;
+
/**
* Constructor when invoked from QL.
*/
@@ -111,8 +112,7 @@
super();
}
- protected static String getResourceFiles(Configuration conf,
- SessionState.ResourceType t) {
+ protected static String getResourceFiles(Configuration conf, SessionState.ResourceType t) {
// fill in local files to be added to the task environment
SessionState ss = SessionState.get();
Set files = (ss == null) ? null : ss.list_resource(t, null);
@@ -122,8 +122,8 @@
try {
realFiles.add(Utilities.realFile(one, conf));
} catch (IOException e) {
- throw new RuntimeException("Cannot validate file " + one
- + "due to exception: " + e.getMessage(), e);
+ throw new RuntimeException("Cannot validate file " + one + "due to exception: "
+ + e.getMessage(), e);
}
}
return StringUtils.join(realFiles, ",");
@@ -143,8 +143,7 @@
* Initialization when invoked from QL.
*/
@Override
- public void initialize(HiveConf conf, QueryPlan queryPlan,
- DriverContext driverContext) {
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) {
super.initialize(conf, queryPlan, driverContext);
job = new JobConf(conf, ExecDriver.class);
@@ -164,8 +163,7 @@
if (StringUtils.isNotBlank(addedJars)) {
HiveConf.setVar(job, ConfVars.HIVEADDEDJARS, addedJars);
}
- String addedArchives = getResourceFiles(job,
- SessionState.ResourceType.ARCHIVE);
+ String addedArchives = getResourceFiles(job, SessionState.ResourceType.ARCHIVE);
if (StringUtils.isNotBlank(addedArchives)) {
HiveConf.setVar(job, ConfVars.HIVEADDEDARCHIVES, addedArchives);
}
@@ -182,24 +180,24 @@
}
/**
- * A list of the currently running jobs spawned in this Hive instance that is
- * used to kill all running jobs in the event of an unexpected shutdown -
- * i.e., the JVM shuts down while there are still jobs running.
+ * A list of the currently running jobs spawned in this Hive instance that is used to kill all
+ * running jobs in the event of an unexpected shutdown - i.e., the JVM shuts down while there are
+ * still jobs running.
*/
- private static Map runningJobKillURIs =
- Collections.synchronizedMap(new HashMap());
+ private static Map runningJobKillURIs = Collections
+ .synchronizedMap(new HashMap());
/**
- * In Hive, when the user control-c's the command line, any running jobs
- * spawned from that command line are best-effort killed.
+ * In Hive, when the user control-c's the command line, any running jobs spawned from that command
+ * line are best-effort killed.
*
- * This static constructor registers a shutdown thread to iterate over all the
- * running job kill URLs and do a get on them.
+ * This static constructor registers a shutdown thread to iterate over all the running job kill
+ * URLs and do a get on them.
*
*/
static {
- if (new org.apache.hadoop.conf.Configuration().getBoolean(
- "webinterface.private.actions", false)) {
+ if (new org.apache.hadoop.conf.Configuration()
+ .getBoolean("webinterface.private.actions", false)) {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
@@ -207,13 +205,13 @@
for (String uri : runningJobKillURIs.values()) {
try {
System.err.println("killing job with: " + uri);
- java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(
- uri).openConnection();
+ java.net.HttpURLConnection conn = (java.net.HttpURLConnection) new java.net.URL(uri)
+ .openConnection();
conn.setRequestMethod("POST");
int retCode = conn.getResponseCode();
if (retCode != 200) {
- System.err.println("Got an error trying to kill job with URI: "
- + uri + " = " + retCode);
+ System.err.println("Got an error trying to kill job with URI: " + uri + " = "
+ + retCode);
}
} catch (Exception e) {
System.err.println("trying to kill job, caught: " + e);
@@ -235,22 +233,19 @@
} else {
String hp = job.get("mapred.job.tracker");
if (SessionState.get() != null) {
- SessionState.get().getHiveHistory().setTaskProperty(
- SessionState.get().getQueryId(), getId(), Keys.TASK_HADOOP_ID,
- rj.getJobID());
+ SessionState.get().getHiveHistory().setTaskProperty(SessionState.get().getQueryId(),
+ getId(), Keys.TASK_HADOOP_ID, rj.getJobID());
}
- console.printInfo(ExecDriver.getJobStartMsg(rj.getJobID())
- + ", Tracking URL = " + rj.getTrackingURL());
- console.printInfo("Kill Command = "
- + HiveConf.getVar(job, HiveConf.ConfVars.HADOOPBIN)
+ console.printInfo(ExecDriver.getJobStartMsg(rj.getJobID()) + ", Tracking URL = "
+ + rj.getTrackingURL());
+ console.printInfo("Kill Command = " + HiveConf.getVar(job, HiveConf.ConfVars.HADOOPBIN)
+ " job -Dmapred.job.tracker=" + hp + " -kill " + rj.getJobID());
}
}
/**
- * This class contains the state of the running task Going forward, we will
- * return this handle from execute and Driver can split execute into start,
- * monitorProgess and postProcess.
+ * This class contains the state of the running task Going forward, we will return this handle
+ * from execute and Driver can split execute into start, monitorProgess and postProcess.
*/
private static class ExecDriverTaskHandle extends TaskHandle {
JobClient jc;
@@ -280,16 +275,14 @@
}
/**
- * Fatal errors are those errors that cannot be recovered by retries. These
- * are application dependent. Examples of fatal errors include: - the small
- * table in the map-side joins is too large to be feasible to be handled by
- * one mapper. The job should fail and the user should be warned to use
- * regular joins rather than map-side joins. Fatal errors are indicated by
- * counters that are set at execution time. If the counter is non-zero, a
- * fatal error occurred. The value of the counter indicates the error type.
+ * Fatal errors are those errors that cannot be recovered by retries. These are application
+ * dependent. Examples of fatal errors include: - the small table in the map-side joins is too
+ * large to be feasible to be handled by one mapper. The job should fail and the user should be
+ * warned to use regular joins rather than map-side joins. Fatal errors are indicated by counters
+ * that are set at execution time. If the counter is non-zero, a fatal error occurred. The value
+ * of the counter indicates the error type.
*
- * @return true if fatal errors happened during job execution, false
- * otherwise.
+ * @return true if fatal errors happened during job execution, false otherwise.
*/
private boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg) {
if (ctrs == null) {
@@ -299,7 +292,7 @@
}
// check for number of created files
long numFiles = ctrs.getCounter(ProgressCounter.CREATED_FILES);
- long upperLimit = HiveConf.getLongVar(job, HiveConf.ConfVars.MAXCREATEDFILES);
+ long upperLimit = HiveConf.getLongVar(job, HiveConf.ConfVars.MAXCREATEDFILES);
if (numFiles > upperLimit) {
errMsg.append("total number of created files exceeds ").append(upperLimit);
return true;
@@ -335,8 +328,7 @@
} catch (InterruptedException e) {
}
- if (initializing &&
- ShimLoader.getHadoopShims().isJobPreparing(rj)) {
+ if (initializing && ShimLoader.getHadoopShims().isJobPreparing(rj)) {
// No reason to poll untill the job is initialized
continue;
} else {
@@ -366,8 +358,7 @@
Counters ctrs = th.getCounters();
if (fatal = checkFatalErrors(ctrs, errMsg)) {
- console.printError("[Fatal Error] " + errMsg.toString()
- + ". Killing the job.");
+ console.printError("[Fatal Error] " + errMsg.toString() + ". Killing the job.");
rj.killJob();
continue;
}
@@ -375,8 +366,8 @@
updateCounters(ctrs, rj);
- String report = " " + getId() + " map = " + mapProgress + "%, reduce = "
- + reduceProgress + "%";
+ String report = " " + getId() + " map = " + mapProgress + "%, reduce = " + reduceProgress
+ + "%";
if (!report.equals(lastReport)
|| System.currentTimeMillis() >= reportTime + maxReportInterval) {
@@ -386,12 +377,10 @@
String output = dateFormat.format(Calendar.getInstance().getTime()) + report;
SessionState ss = SessionState.get();
if (ss != null) {
- ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(),
- getId(), ctrs);
- ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(),
- getId(), Keys.TASK_HADOOP_PROGRESS, output);
- ss.getHiveHistory().progressTask(SessionState.get().getQueryId(),
- this);
+ ss.getHiveHistory().setTaskCounters(SessionState.get().getQueryId(), getId(), ctrs);
+ ss.getHiveHistory().setTaskProperty(SessionState.get().getQueryId(), getId(),
+ Keys.TASK_HADOOP_PROGRESS, output);
+ ss.getHiveHistory().progressTask(SessionState.get().getQueryId(), this);
ss.getHiveHistory().logPlanProgress(queryPlan);
}
console.printInfo(output);
@@ -411,7 +400,7 @@
if (checkFatalErrors(ctrs, errMsg)) {
console.printError("[Fatal Error] " + errMsg.toString());
success = false;
- } else {
+ } else {
success = rj.isSuccessful();
}
}
@@ -434,10 +423,8 @@
private void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
mapProgress = Math.round(rj.mapProgress() * 100);
reduceProgress = Math.round(rj.reduceProgress() * 100);
- taskCounters.put("CNTR_NAME_" + getId() + "_MAP_PROGRESS", Long
- .valueOf(mapProgress));
- taskCounters.put("CNTR_NAME_" + getId() + "_REDUCE_PROGRESS", Long
- .valueOf(reduceProgress));
+ taskCounters.put("CNTR_NAME_" + getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress));
+ taskCounters.put("CNTR_NAME_" + getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress));
if (ctrs == null) {
// hadoop might return null if it cannot locate the job.
// we may still be able to retrieve the job status - so ignore
@@ -512,8 +499,8 @@
job.setMapOutputValueClass(BytesWritable.class);
try {
- job.setPartitionerClass((Class extends Partitioner>)
- (Class.forName(HiveConf.getVar(job, HiveConf.ConfVars.HIVEPARTITIONER))));
+ job.setPartitionerClass((Class extends Partitioner>) (Class.forName(HiveConf.getVar(job,
+ HiveConf.ConfVars.HIVEPARTITIONER))));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.getMessage());
}
@@ -522,8 +509,8 @@
job.setNumMapTasks(work.getNumMapTasks().intValue());
}
if (work.getMinSplitSize() != null) {
- HiveConf.setIntVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE,
- work.getMinSplitSize().intValue());
+ HiveConf.setIntVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work.getMinSplitSize()
+ .intValue());
}
job.setNumReduceTasks(work.getNumReduceTasks().intValue());
job.setReducerClass(ExecReducer.class);
@@ -533,11 +520,9 @@
}
// Turn on speculative execution for reducers
- boolean useSpeculativeExecReducers =
- HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVESPECULATIVEEXECREDUCERS);
- HiveConf.setBoolVar(
- job,
- HiveConf.ConfVars.HADOOPSPECULATIVEEXECREDUCERS,
+ boolean useSpeculativeExecReducers = HiveConf.getBoolVar(job,
+ HiveConf.ConfVars.HIVESPECULATIVEEXECREDUCERS);
+ HiveConf.setBoolVar(job, HiveConf.ConfVars.HADOOPSPECULATIVEEXECREDUCERS,
useSpeculativeExecReducers);
String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT);
@@ -548,8 +533,7 @@
LOG.info("Using " + inpFormat);
try {
- job.setInputFormat((Class extends InputFormat>) (Class
- .forName(inpFormat)));
+ job.setInputFormat((Class extends InputFormat>) (Class.forName(inpFormat)));
} catch (ClassNotFoundException e) {
throw new RuntimeException(e.getMessage());
}
@@ -563,8 +547,9 @@
String auxJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEAUXJARS);
String addedJars = HiveConf.getVar(job, HiveConf.ConfVars.HIVEADDEDJARS);
if (StringUtils.isNotBlank(auxJars) || StringUtils.isNotBlank(addedJars)) {
- String allJars = StringUtils.isNotBlank(auxJars) ? (StringUtils
- .isNotBlank(addedJars) ? addedJars + "," + auxJars : auxJars)
+ String allJars = StringUtils.isNotBlank(auxJars) ? (StringUtils.isNotBlank(addedJars) ? addedJars
+ + "," + auxJars
+ : auxJars)
: addedJars;
LOG.info("adding libjars: " + allJars);
initializeFiles("tmpjars", allJars);
@@ -576,8 +561,7 @@
initializeFiles("tmpfiles", addedFiles);
}
// Transfer HIVEADDEDARCHIVES to "tmparchives" so hadoop understands it
- String addedArchives = HiveConf.getVar(job,
- HiveConf.ConfVars.HIVEADDEDARCHIVES);
+ String addedArchives = HiveConf.getVar(job, HiveConf.ConfVars.HIVEADDEDARCHIVES);
if (StringUtils.isNotBlank(addedArchives)) {
initializeFiles("tmparchives", addedArchives);
}
@@ -585,21 +569,18 @@
int returnVal = 0;
RunningJob rj = null;
- boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
- HiveConf.ConfVars.HADOOPJOBNAME));
+ boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJOBNAME));
if (noName) {
// This is for a special case to ensure unit tests pass
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB"
- + Utilities.randGen.nextInt());
+ HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB" + Utilities.randGen.nextInt());
}
try {
- //propagate the file to distributed cache
- MapredLocalWork localwork =work.getMapLocalWork();
- if(localwork != null){
+ // propagate the file to distributed cache
+ MapredLocalWork localwork = work.getMapLocalWork();
+ if (localwork != null) {
boolean localMode = HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local");
- if(!localMode){
- //LOG.info("=========Adding files to HDFS ================");
+ if (!localMode) {
Path localPath = new Path(localwork.getTmpFileURI());
Path hdfsPath = new Path(work.getTmpHDFSFileURI());
@@ -609,28 +590,24 @@
for (int i = 0; i < hashtableFiles.length; i++) {
FileStatus file = hashtableFiles[i];
Path path = file.getPath();
- String fileName=path.getName();
- String hdfsFile = hdfsPath+Path.SEPARATOR+fileName;
+ String fileName = path.getName();
+ String hdfsFile = hdfsPath + Path.SEPARATOR + fileName;
LOG.info("Upload 1 HashTable from" + path + " to: " + hdfsFile);
Path hdfsFilePath = new Path(hdfsFile);
- //hdfs.setVerifyChecksum(false);
- hdfs.copyFromLocalFile(path,hdfsFilePath );
- short replication = (short)job.getInt("mapred.submit.replication", 10);
+ hdfs.copyFromLocalFile(path, hdfsFilePath);
+ short replication = (short) job.getInt("mapred.submit.replication", 10);
hdfs.setReplication(hdfsFilePath, replication);
}
- //LOG.info("=========Adding files to distributed cache================");
-
FileStatus[] hashtableRemoteFiles = hdfs.listStatus(hdfsPath);
for (int i = 0; i < hashtableRemoteFiles.length; i++) {
FileStatus file = hashtableRemoteFiles[i];
Path path = file.getPath();
DistributedCache.addCacheFile(path.toUri(), job);
- LOG.info("add 1 hashtable file to distributed cache: "
- + path.toUri());
+
+ LOG.info("add 1 hashtable file to distributed cache: " + path.toUri());
}
- //LOG.info("===========Finishing adding files to distributed cache==========");
}
}
@@ -670,8 +647,7 @@
// add to list of running jobs to kill in case of abnormal shutdown
- runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL()
- + "&action=kill");
+ runningJobKillURIs.put(rj.getJobID(), rj.getTrackingURL() + "&action=kill");
ExecDriverTaskHandle th = new ExecDriverTaskHandle(jc, rj);
jobInfo(rj);
@@ -701,15 +677,14 @@
// Has to use full name to make sure it does not conflict with
// org.apache.commons.lang.StringUtils
- console.printError(mesg, "\n"
- + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ console.printError(mesg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
success = false;
returnVal = 1;
} finally {
Utilities.clearMapRedWork(job);
try {
- if(ctxCreated) {
+ if (ctxCreated) {
ctx.clear();
}
@@ -728,8 +703,7 @@
if (rj != null) {
JobCloseFeedBack feedBack = new JobCloseFeedBack();
if (work.getAliasToWork() != null) {
- for (Operator extends Serializable> op : work.getAliasToWork()
- .values()) {
+ for (Operator extends Serializable> op : work.getAliasToWork().values()) {
op.jobClose(job, success, feedBack);
}
}
@@ -742,10 +716,8 @@
if (success) {
success = false;
returnVal = 3;
- String mesg = "Job Commit failed with exception '"
- + Utilities.getNameMessage(e) + "'";
- console.printError(mesg, "\n"
- + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ String mesg = "Job Commit failed with exception '" + Utilities.getNameMessage(e) + "'";
+ console.printError(mesg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
}
}
@@ -772,8 +744,7 @@
return "Ended Job = " + jobId;
}
- private String getTaskAttemptLogUrl(String taskTrackerHttpAddress,
- String taskAttemptId) {
+ private String getTaskAttemptLogUrl(String taskTrackerHttpAddress, String taskAttemptId) {
return taskTrackerHttpAddress + "/tasklog?taskid=" + taskAttemptId + "&all=true";
}
@@ -786,12 +757,15 @@
this.jobId = jobId;
logUrls = new HashSet();
}
+
public void addLogUrl(String logUrl) {
logUrls.add(logUrl);
}
+
public HashSet getLogUrls() {
return logUrls;
}
+
public String getJobId() {
return jobId;
}
@@ -811,8 +785,7 @@
// Loop to get all task completion events because getTaskCompletionEvents
// only returns a subset per call
while (true) {
- TaskCompletionEvent[] taskCompletions =
- rj.getTaskCompletionEvents(startIndex);
+ TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex);
if (taskCompletions == null || taskCompletions.length == 0) {
break;
@@ -836,14 +809,13 @@
String jobId = taskJobIds[1];
TaskInfo ti = taskIdToInfo.get(taskId);
- if(ti == null) {
+ if (ti == null) {
ti = new TaskInfo(jobId);
taskIdToInfo.put(taskId, ti);
}
// These tasks should have come from the same job.
- assert(ti.getJobId() == jobId);
- ti.getLogUrls().add(
- getTaskAttemptLogUrl(t.getTaskTrackerHttp(), t.getTaskId()));
+ assert (ti.getJobId() == jobId);
+ ti.getLogUrls().add(getTaskAttemptLogUrl(t.getTaskTrackerHttp(), t.getTaskId()));
// If a task failed, then keep track of the total number of failures
// for that task (typically, a task gets re-run up to 4 times if it
@@ -889,11 +861,10 @@
if (failures.get(task).intValue() == maxFailures) {
TaskInfo ti = taskIdToInfo.get(task);
String jobId = ti.getJobId();
- String taskUrl = jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" +
- task.toString();
+ String taskUrl = jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
TaskLogProcessor tlp = new TaskLogProcessor(conf);
- for(String logUrl : ti.getLogUrls()) {
+ for (String logUrl : ti.getLogUrls()) {
tlp.addTaskAttemptLogUrl(logUrl);
}
@@ -910,7 +881,7 @@
sb.append("Task ID:\n " + task + "\n\n");
sb.append("URL:\n " + taskUrl + "\n");
- for(ErrorAndSolution e : errors) {
+ for (ErrorAndSolution e : errors) {
sb.append("\n");
sb.append("Possible error:\n " + e.getError() + "\n\n");
sb.append("Solution:\n " + e.getSolution() + "\n");
@@ -934,36 +905,29 @@
}
/**
- * we are running the hadoop job via a sub-command. this typically
- * happens when we are running jobs in local mode. the log4j in this
- * mode is controlled as follows:
- * 1. if the admin provides a log4j properties file especially for
- * execution mode - then we pick that up
- * 2. otherwise - we default to the regular hive log4j properties if
- * one is supplied
- * 3. if none of the above two apply - we don't do anything - the log4j
- * properties would likely be determined by hadoop.
+ * we are running the hadoop job via a sub-command. this typically happens when we are running
+ * jobs in local mode. the log4j in this mode is controlled as follows: 1. if the admin provides a
+ * log4j properties file especially for execution mode - then we pick that up 2. otherwise - we
+ * default to the regular hive log4j properties if one is supplied 3. if none of the above two
+ * apply - we don't do anything - the log4j properties would likely be determined by hadoop.
*
- * The intention behind providing a separate option #1 is to be able to
- * collect hive run time logs generated in local mode in a separate
- * (centralized) location if desired. This mimics the behavior of hive
- * run time logs when running against a hadoop cluster where they are available
+ * The intention behind providing a separate option #1 is to be able to collect hive run time logs
+ * generated in local mode in a separate (centralized) location if desired. This mimics the
+ * behavior of hive run time logs when running against a hadoop cluster where they are available
* on the tasktracker nodes.
*/
private static void setupChildLog4j(Configuration conf) {
URL hive_l4j = ExecDriver.class.getClassLoader().getResource(SessionState.HIVE_EXEC_L4J);
- if(hive_l4j == null) {
- hive_l4j = ExecDriver.class.getClassLoader().getResource
- (SessionState.HIVE_L4J);
+ if (hive_l4j == null) {
+ hive_l4j = ExecDriver.class.getClassLoader().getResource(SessionState.HIVE_L4J);
}
if (hive_l4j != null) {
// setting queryid so that log4j configuration can use it to generate
// per query log file
- System.setProperty
- (HiveConf.ConfVars.HIVEQUERYID.toString(),
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID));
+ System.setProperty(HiveConf.ConfVars.HIVEQUERYID.toString(), HiveConf.getVar(conf,
+ HiveConf.ConfVars.HIVEQUERYID));
LogManager.resetConfiguration();
PropertyConfigurator.configure(hive_l4j);
}
@@ -975,8 +939,7 @@
ArrayList jobConfArgs = new ArrayList();
boolean noLog = false;
String files = null;
- boolean localtask=false;
-
+ boolean localtask = false;
try {
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-plan")) {
@@ -987,7 +950,7 @@
noLog = true;
} else if (args[i].equals("-files")) {
files = args[++i];
- }else if(args[i].equals("-localtask")) {
+ } else if (args[i].equals("-localtask")) {
localtask = true;
}
}
@@ -996,11 +959,11 @@
printUsage();
}
- JobConf conf ;
- if(localtask) {
- conf= new JobConf(MapredLocalTask.class);
+ JobConf conf;
+ if (localtask) {
+ conf = new JobConf(MapredLocalTask.class);
} else {
- conf= new JobConf(ExecDriver.class);
+ conf = new JobConf(ExecDriver.class);
}
StringBuilder sb = new StringBuilder("JobConf:\n");
@@ -1013,8 +976,8 @@
conf.set(key, value);
sb.append(key).append("=").append(value).append("\n");
} catch (UnsupportedEncodingException e) {
- System.err.println("Unexpected error " + e.getMessage()
- + " while encoding " + one.substring(eqIndex + 1));
+ System.err.println("Unexpected error " + e.getMessage() + " while encoding "
+ + one.substring(eqIndex + 1));
System.exit(3);
}
}
@@ -1045,10 +1008,10 @@
// print out the location of the log file for the user so
// that it's easy to find reason for local mode execution failures
- for (Appender appender: Collections.list
- ((Enumeration)LogManager.getRootLogger().getAllAppenders())) {
+ for (Appender appender : Collections.list((Enumeration) LogManager.getRootLogger()
+ .getAllAppenders())) {
if (appender instanceof FileAppender) {
- console.printInfo("Execution log at: " + ((FileAppender)appender).getFile());
+ console.printInfo("Execution log at: " + ((FileAppender) appender).getFile());
}
}
@@ -1085,11 +1048,11 @@
throw new HiveException(e.getMessage(), e);
}
int ret;
- if(localtask) {
+ if (localtask) {
memoryMXBean = ManagementFactory.getMemoryMXBean();
MapredLocalWork plan = Utilities.deserializeMapRedLocalWork(pathData, conf);
MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent);
- ret= ed.executeFromChildJVM(new DriverContext());
+ ret = ed.executeFromChildJVM(new DriverContext());
} else {
MapredWork plan = Utilities.deserializeMapRedWork(pathData, conf);
@@ -1103,8 +1066,8 @@
}
/**
- * Given a Hive Configuration object - generate a command line fragment for
- * passing such configuration information to ExecDriver.
+ * Given a Hive Configuration object - generate a command line fragment for passing such
+ * configuration information to ExecDriver.
*/
public static String generateCmdLine(HiveConf hconf) {
try {
@@ -1117,8 +1080,7 @@
for (Object one : deltaP.keySet()) {
String oneProp = (String) one;
- if (hadoopLocalMode
- && (oneProp.equals(hadoopSysDir) || oneProp.equals(hadoopWorkDir))) {
+ if (hadoopLocalMode && (oneProp.equals(hadoopSysDir) || oneProp.equals(hadoopWorkDir))) {
continue;
}
@@ -1139,15 +1101,15 @@
sb.append("-jobconf ");
sb.append(hadoopSysDir);
sb.append("=");
- sb.append(URLEncoder.encode(hconf.get(hadoopSysDir) + "/"
- + Utilities.randGen.nextInt(), "UTF-8"));
+ sb.append(URLEncoder.encode(hconf.get(hadoopSysDir) + "/" + Utilities.randGen.nextInt(),
+ "UTF-8"));
sb.append(" ");
sb.append("-jobconf ");
sb.append(hadoopWorkDir);
sb.append("=");
- sb.append(URLEncoder.encode(hconf.get(hadoopWorkDir) + "/"
- + Utilities.randGen.nextInt(), "UTF-8"));
+ sb.append(URLEncoder.encode(hconf.get(hadoopWorkDir) + "/" + Utilities.randGen.nextInt(),
+ "UTF-8"));
}
return sb.toString();
@@ -1170,9 +1132,8 @@
/**
* Handle a empty/null path for a given alias.
*/
- private int addInputPath(String path, JobConf job, MapredWork work,
- String hiveScratchDir, int numEmptyPaths, boolean isEmptyPath,
- String alias) throws Exception {
+ private int addInputPath(String path, JobConf job, MapredWork work, String hiveScratchDir,
+ int numEmptyPaths, boolean isEmptyPath, String alias) throws Exception {
// either the directory does not exist or it is empty
assert path == null || isEmptyPath;
@@ -1184,7 +1145,7 @@
outFileFormat = partDesc.getOutputFileFormatClass();
nonNative = partDesc.getTableDesc().isNonNative();
} else {
- TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
+ TableDesc tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
outFileFormat = tableDesc.getOutputFileFormatClass();
nonNative = tableDesc.isNonNative();
}
@@ -1237,15 +1198,14 @@
work.setPathToPartitionInfo(pathToPartitionInfo);
String onefile = newPath.toString();
- RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(
- job, newFilePath, Text.class, false, new Properties(), null);
+ RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newFilePath,
+ Text.class, false, new Properties(), null);
recWriter.close(false);
FileInputFormat.addInputPaths(job, onefile);
return numEmptyPaths;
}
- private void addInputPaths(JobConf job, MapredWork work, String hiveScratchDir)
- throws Exception {
+ private void addInputPaths(JobConf job, MapredWork work, String hiveScratchDir) throws Exception {
int numEmptyPaths = 0;
List