Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java (revision 1038344) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java (working copy) @@ -255,10 +255,14 @@ Partition part = null; Iterator iter = prunedParts.getConfirmedPartns() .iterator(); - part = iter.next(); + if (iter.hasNext()) { + part = iter.next(); + } if (part == null) { iter = prunedParts.getUnknownPartns().iterator(); - part = iter.next(); + if (iter.hasNext()) { + part = iter.next(); + } } assert part != null; Integer num = new Integer(part.getBucketCount()); Index: ql/src/test/queries/clientpositive/smb_mapjoin9.q =================================================================== --- ql/src/test/queries/clientpositive/smb_mapjoin9.q (revision 0) +++ ql/src/test/queries/clientpositive/smb_mapjoin9.q (revision 0) @@ -0,0 +1,30 @@ +create table hive_test_smb_bucket1 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets; +create table hive_test_smb_bucket2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets; + +set hive.enforce.bucketing = true; +set hive.enforce.sorting = true; + +insert overwrite table hive_test_smb_bucket1 partition (ds='2010-10-15') select key, value from src; +insert overwrite table hive_test_smb_bucket2 partition (ds='2010-10-15') select key, value from src; + +set hive.optimize.bucketmapjoin = true; +set hive.optimize.bucketmapjoin.sortedmerge = true; +set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; + +explain +create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL; + + +create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL; + +drop table smb_mapjoin9_results; +drop table hive_test_smb_bucket1; +drop table hive_test_smb_bucket2; \ No newline at end of file Index: ql/src/test/results/clientpositive/smb_mapjoin9.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out (revision 0) +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out (revision 0) @@ -0,0 +1,187 @@ +PREHOOK: query: create table hive_test_smb_bucket1 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table hive_test_smb_bucket1 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@hive_test_smb_bucket1 +PREHOOK: query: create table hive_test_smb_bucket2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table hive_test_smb_bucket2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@hive_test_smb_bucket2 +PREHOOK: query: insert overwrite table hive_test_smb_bucket1 partition (ds='2010-10-15') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hive_test_smb_bucket1@ds=2010-10-15 +POSTHOOK: query: insert overwrite table hive_test_smb_bucket1 partition (ds='2010-10-15') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hive_test_smb_bucket1@ds=2010-10-15 +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table hive_test_smb_bucket2 partition (ds='2010-10-15') select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@hive_test_smb_bucket2@ds=2010-10-15 +POSTHOOK: query: insert overwrite table hive_test_smb_bucket2 partition (ds='2010-10-15') select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@hive_test_smb_bucket2@ds=2010-10-15 +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain +create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL +POSTHOOK: type: CREATETABLE +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_CREATETABLE smb_mapjoin9_results TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF hive_test_smb_bucket1 a) (TOK_TABREF hive_test_smb_bucket2 b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) key) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) ds)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) k2)) (TOK_WHERE (and (and (= (. (TOK_TABLE_OR_COL a) ds) '2010-10-15') (= (. (TOK_TABLE_OR_COL b) ds) '2010-10-15')) (TOK_FUNCTION TOK_ISNOTNULL (. (TOK_TABLE_OR_COL b) key))))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + a + TableScan + alias: a + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {ds} + 1 {key} {value} {ds} + handleSkewJoin: false + keys: + 0 [Column[key]] + 1 [Column[key]] + outputColumnNames: _col0, _col2, _col5, _col6, _col7 + Position of Big Table: 0 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col2 + type: string + expr: _col5 + type: int + expr: _col6 + type: string + expr: _col7 + type: string + outputColumnNames: _col0, _col2, _col5, _col6, _col7 + Filter Operator + predicate: + expr: (((_col2 = '2010-10-15') and (_col7 = '2010-10-15')) and _col5 is not null) + type: boolean + Select Operator + expressions: + expr: _col5 + type: int + expr: _col6 + type: string + expr: _col7 + type: string + expr: _col0 + type: int + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true + destination: pfile:///Users/heyongqiang/Documents/workspace/Hive-Trunk/build/ql/test/data/warehouse/smb_mapjoin9_results + + Stage: Stage-3 + Create Table Operator: + Create Table + columns: k1 int, value string, ds string, k2 int + if not exists: false + input format: org.apache.hadoop.mapred.TextInputFormat + # buckets: -1 + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + name: smb_mapjoin9_results + isExternal: false + + +PREHOOK: query: create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL +PREHOOK: type: CREATETABLE +PREHOOK: Input: default@hive_test_smb_bucket1@ds=2010-10-15 +PREHOOK: Input: default@hive_test_smb_bucket2@ds=2010-10-15 +POSTHOOK: query: create table smb_mapjoin9_results as +SELECT /* + MAPJOIN(b) */ b.key as k1, b.value, b.ds, a.key as k2 +FROM hive_test_smb_bucket1 a JOIN +hive_test_smb_bucket2 b +ON a.key = b.key WHERE a.ds = '2010-10-15' and b.ds='2010-10-15' and b.key IS NOT NULL +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: default@hive_test_smb_bucket1@ds=2010-10-15 +POSTHOOK: Input: default@hive_test_smb_bucket2@ds=2010-10-15 +POSTHOOK: Output: default@smb_mapjoin9_results +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop table smb_mapjoin9_results +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@smb_mapjoin9_results +PREHOOK: Output: default@smb_mapjoin9_results +POSTHOOK: query: drop table smb_mapjoin9_results +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@smb_mapjoin9_results +POSTHOOK: Output: default@smb_mapjoin9_results +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop table hive_test_smb_bucket1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hive_test_smb_bucket1 +PREHOOK: Output: default@hive_test_smb_bucket1 +POSTHOOK: query: drop table hive_test_smb_bucket1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hive_test_smb_bucket1 +POSTHOOK: Output: default@hive_test_smb_bucket1 +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: drop table hive_test_smb_bucket2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@hive_test_smb_bucket2 +PREHOOK: Output: default@hive_test_smb_bucket2 +POSTHOOK: query: drop table hive_test_smb_bucket2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@hive_test_smb_bucket2 +POSTHOOK: Output: default@hive_test_smb_bucket2 +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket1 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: hive_test_smb_bucket2 PARTITION(ds=2010-10-15).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]