Index: src/test/results/clientpositive/join29.q.out =================================================================== --- src/test/results/clientpositive/join29.q.out (revision 1032800) +++ src/test/results/clientpositive/join29.q.out (working copy) @@ -89,12 +89,12 @@ Stage: Stage-9 Map Reduce Local Work Alias -> Map Local Tables: - file:/tmp/njain/hive_2010-10-27_11-54-34_757_9120981059274795938/-mr-10004 + file:/tmp/liyintang/hive_2010-11-04_12-10-25_127_7262353630093457446/-mr-10004 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-54-34_757_9120981059274795938/-mr-10004 - JDBM Sink Operator + file:/tmp/liyintang/hive_2010-11-04_12-10-25_127_7262353630093457446/-mr-10004 + HashTable Sink Operator condition expressions: 0 {_col0} {_col1} 1 {_col1} @@ -107,7 +107,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-54-34_757_9120981059274795938/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-10-25_127_7262353630093457446/-mr-10002 Map Join Operator condition map: Inner Join 0 to 1 @@ -165,7 +165,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-54-34_757_9120981059274795938/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-10-25_127_7262353630093457446/-ext-10000 Stage: Stage-0 Move Operator @@ -183,7 +183,7 @@ Stage: Stage-4 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-54-34_757_9120981059274795938/-ext-10003 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-10-25_127_7262353630093457446/-ext-10003 File Output Operator compressed: false GlobalTableId: 0 @@ -272,11 +272,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-46_357_4458958652114796552/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-10-42_703_1867902746100829562/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-46_357_4458958652114796552/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-10-42_703_1867902746100829562/-mr-10000 POSTHOOK: Lineage: dest_j1.cnt1 EXPRESSION [(src1)x.null, ] POSTHOOK: Lineage: dest_j1.cnt2 EXPRESSION [(src)y.null, ] POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] Index: src/test/results/clientpositive/join38.q.out =================================================================== --- src/test/results/clientpositive/join38.q.out (revision 1032800) +++ src/test/results/clientpositive/join38.q.out (working copy) @@ -26,11 +26,11 @@ PREHOOK: query: select * from tmp PREHOOK: type: QUERY PREHOOK: Input: default@tmp -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-04_133_1536500479311783598/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-52_842_1951454920482464680/-mr-10000 POSTHOOK: query: select * from tmp POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-04_133_1536500479311783598/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-52_842_1951454920482464680/-mr-10000 POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] @@ -89,7 +89,7 @@ a TableScan alias: a - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {value} 1 {col5} {col11} @@ -133,7 +133,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-58-04_322_8814213860188537568/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-13-53_269_4711133131177071929/-mr-10002 Select Operator expressions: expr: _col1 @@ -221,7 +221,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@tmp -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-04_499_6020968433330491483/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-53_398_7677851135191109830/-mr-10000 POSTHOOK: query: FROM src a JOIN tmp b ON (a.key = b.col11) SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count where b.col11 = 111 @@ -229,7 +229,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@tmp -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-04_499_6020968433330491483/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-53_398_7677851135191109830/-mr-10000 POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] Index: src/test/results/clientpositive/bucketmapjoin5.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin5.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin5.q.out (working copy) @@ -134,7 +134,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -147,16 +147,16 @@ Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket20.txt], srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket20.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket21.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket20.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket21.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket22.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket23.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket20.txt 0 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket21.txt 1 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket22.txt 2 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket23.txt 3 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket22.txt 2 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09/srcbucket23.txt 3 Stage: Stage-1 Map Reduce @@ -198,9 +198,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -210,12 +210,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213068 + transient_lastDdlTime 1288895099 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -225,10 +225,10 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09 [b] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -242,13 +242,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213061 + transient_lastDdlTime 1288895084 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -260,17 +260,17 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213061 + transient_lastDdlTime 1288895084 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part name: srcbucket_mapjoin_part - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -284,13 +284,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213061 + transient_lastDdlTime 1288895084 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -302,13 +302,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213061 + transient_lastDdlTime 1288895084 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part name: srcbucket_mapjoin_part @@ -320,14 +320,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002 - destination: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -337,28 +337,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213068 + transient_lastDdlTime 1288895099 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -369,12 +369,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213068 + transient_lastDdlTime 1288895099 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -382,9 +382,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002 [pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-57-48_658_4685912752018510968/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-24-59_289_127678237669643408/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -395,12 +395,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213068 + transient_lastDdlTime 1288895099 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -411,12 +411,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213068 + transient_lastDdlTime 1288895099 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -446,11 +446,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-00_476_6141696786840834340/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-25-24_662_1799707746593499124/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-00_476_6141696786840834340/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-25-24_662_1799707746593499124/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part)b.FieldSchema(name:value, type:string, comment:null), ] @@ -501,11 +501,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-16_764_7387427002932242988/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-25-57_995_7569034646613062387/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-16_764_7387427002932242988/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-25-57_995_7569034646613062387/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -544,14 +544,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-24_273_8996797782903684116/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-26-10_276_2123427496583071251/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-24_273_8996797782903684116/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-26-10_276_2123427496583071251/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -613,7 +613,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -626,12 +626,12 @@ Alias Bucket Base File Name Mapping: a {srcbucket22.txt=[srcbucket20.txt], srcbucket23.txt=[srcbucket21.txt], ds=2008-04-09/srcbucket22.txt=[srcbucket20.txt], ds=2008-04-09/srcbucket23.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket22.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket23.txt=[pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket22.txt 0 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket23.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket22.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09/srcbucket23.txt 1 Stage: Stage-1 Map Reduce @@ -673,9 +673,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -685,7 +685,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -694,7 +694,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17966 - transient_lastDdlTime 1288213096 + transient_lastDdlTime 1288895157 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -704,10 +704,10 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [b] - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09 [b] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -721,13 +721,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213066 + transient_lastDdlTime 1288895094 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -739,17 +739,17 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213066 + transient_lastDdlTime 1288895094 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part_2 name: srcbucket_mapjoin_part_2 - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -763,13 +763,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213066 + transient_lastDdlTime 1288895094 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -781,13 +781,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213066 + transient_lastDdlTime 1288895094 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part_2 name: srcbucket_mapjoin_part_2 @@ -799,14 +799,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002 - destination: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -816,7 +816,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -825,23 +825,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17966 - transient_lastDdlTime 1288213096 + transient_lastDdlTime 1288895157 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -852,7 +852,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -861,7 +861,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17966 - transient_lastDdlTime 1288213096 + transient_lastDdlTime 1288895157 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -869,9 +869,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002 [pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-58-27_067_2152368000902411635/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-26-15_858_2943817650725873010/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -882,7 +882,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -891,7 +891,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17966 - transient_lastDdlTime 1288213096 + transient_lastDdlTime 1288895157 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -902,7 +902,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -911,7 +911,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17966 - transient_lastDdlTime 1288213096 + transient_lastDdlTime 1288895157 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -953,11 +953,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-35_836_7705379100464575308/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-26-33_961_2572869140765524365/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-35_836_7705379100464575308/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-26-33_961_2572869140765524365/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -1044,11 +1044,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-49_907_5039198453352841568/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-27-04_712_209753154817947520/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-49_907_5039198453352841568/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-27-04_712_209753154817947520/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] @@ -1111,14 +1111,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-56_380_2556535610013066769/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-27-16_508_7488072457189411491/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_13-58-56_380_2556535610013066769/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-27-16_508_7488072457189411491/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] Index: src/test/results/clientpositive/join33.q.out =================================================================== --- src/test/results/clientpositive/join33.q.out (revision 1032800) +++ src/test/results/clientpositive/join33.q.out (working copy) @@ -37,7 +37,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -69,7 +69,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/sdong/hive_2010-10-29_15-39-12_146_3567761358225891739/-mr-10002 + directory: file:/tmp/liyintang/hive_2010-11-08_19-01-43_620_231968462576941228/-mr-10002 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -85,9 +85,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src [y] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src [y] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -98,12 +98,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389296 + transient_lastDdlTime 1289268011 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -114,12 +114,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389296 + transient_lastDdlTime 1289268011 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src name: src @@ -127,7 +127,7 @@ Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2010-10-29_15-39-12_146_3567761358225891739/-mr-10002 + file:/tmp/liyintang/hive_2010-11-08_19-01-43_620_231968462576941228/-mr-10002 Select Operator expressions: expr: _col0 @@ -179,10 +179,10 @@ type: string Needs Tagging: true Path -> Alias: - file:/tmp/sdong/hive_2010-10-29_15-39-12_146_3567761358225891739/-mr-10002 [file:/tmp/sdong/hive_2010-10-29_15-39-12_146_3567761358225891739/-mr-10002] - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] + file:/tmp/liyintang/hive_2010-11-08_19-01-43_620_231968462576941228/-mr-10002 [file:/tmp/liyintang/hive_2010-11-08_19-01-43_620_231968462576941228/-mr-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] Path -> Partition: - file:/tmp/sdong/hive_2010-10-29_15-39-12_146_3567761358225891739/-mr-10002 + file:/tmp/liyintang/hive_2010-11-08_19-01-43_620_231968462576941228/-mr-10002 Partition base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -198,7 +198,7 @@ columns _col0,_col1,_col5 columns.types string,string,string escape.delim \ - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat @@ -212,13 +212,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -229,13 +229,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart name: srcpart @@ -260,9 +260,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-39-12_146_3567761358225891739/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-43_620_231968462576941228/-ext-10000 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-39-12_146_3567761358225891739/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-43_620_231968462576941228/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -272,12 +272,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391952 + transient_lastDdlTime 1289271703 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -288,7 +288,7 @@ Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-39-12_146_3567761358225891739/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-43_620_231968462576941228/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -298,19 +298,19 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391952 + transient_lastDdlTime 1289271703 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-39-12_146_3567761358225891739/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-43_620_231968462576941228/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-39-12_146_3567761358225891739/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-43_620_231968462576941228/-ext-10000/ PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -337,11 +337,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-39-20_686_8959230285981219318/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-01-57_945_6756602084312764446/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-39-20_686_8959230285981219318/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-01-57_945_6756602084312764446/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/mapjoin_mapjoin.q.out =================================================================== --- src/test/results/clientpositive/mapjoin_mapjoin.q.out (revision 1032800) +++ src/test/results/clientpositive/mapjoin_mapjoin.q.out (working copy) @@ -23,7 +23,7 @@ src TableScan alias: src - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -70,7 +70,7 @@ src1 TableScan alias: src1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col0} 1 @@ -83,7 +83,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_14-46-32_639_5385690485575219943/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-50-21_401_1192194208773335423/-mr-10002 Select Operator expressions: expr: _col0 @@ -151,7 +151,7 @@ src TableScan alias: src - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {ds} 1 @@ -198,7 +198,7 @@ src1 TableScan alias: src1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col2} 1 @@ -211,7 +211,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_14-46-32_758_5972659942965972745/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-50-21_619_1359331938520049914/-mr-10002 Select Operator expressions: expr: _col0 @@ -243,7 +243,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_14-46-32_758_5972659942965972745/-mr-10003 + file:/tmp/liyintang/hive_2010-11-04_12-50-21_619_1359331938520049914/-mr-10003 Select Operator expressions: expr: _col2 @@ -309,7 +309,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_14-46-32_877_634052141369406072/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-50-21_821_3740258359127333333/-mr-10000 POSTHOOK: query: select /*+MAPJOIN(src, src1) */ count(*) from srcpart join src src on (srcpart.value=src.value) join src src1 on (srcpart.key=src1.key) group by ds POSTHOOK: type: QUERY POSTHOOK: Input: default@src @@ -317,6 +317,6 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_14-46-32_877_634052141369406072/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-50-21_821_3740258359127333333/-mr-10000 5308 5308 Index: src/test/results/clientpositive/join39.q.out =================================================================== --- src/test/results/clientpositive/join39.q.out (revision 1032800) +++ src/test/results/clientpositive/join39.q.out (working copy) @@ -51,7 +51,7 @@ expr: value type: string outputColumnNames: _col0, _col1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {_col0} {_col1} @@ -119,7 +119,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-58-12_180_546438697212415855/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-14-05_003_6546778973100053760/-ext-10000 Stage: Stage-0 Move Operator @@ -137,7 +137,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-58-12_180_546438697212415855/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-14-05_003_6546778973100053760/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -167,11 +167,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-17_368_425948306842908675/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-14_819_2106628094710415660/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-17_368_425948306842908675/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-14_819_2106628094710415660/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.key1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/join25.q.out =================================================================== --- src/test/results/clientpositive/join25.q.out (revision 1032800) +++ src/test/results/clientpositive/join25.q.out (working copy) @@ -36,7 +36,7 @@ x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -109,7 +109,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-58_967_6806228839153646653/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-09-26_418_6554875518928365288/-ext-10000 Stage: Stage-0 Move Operator @@ -127,7 +127,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-58_967_6806228839153646653/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-09-26_418_6554875518928365288/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -158,11 +158,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-04_814_7186456825570807641/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-09-36_127_7873084918847057156/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-04_814_7186456825570807641/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-09-36_127_7873084918847057156/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/join34.q.out =================================================================== --- src/test/results/clientpositive/join34.q.out (revision 1032800) +++ src/test/results/clientpositive/join34.q.out (working copy) @@ -47,7 +47,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col1} 1 {key} {value} @@ -115,9 +115,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -127,12 +127,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -193,9 +193,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -205,12 +205,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -220,9 +220,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src [null-subquery1:subq1-subquery1:x, null-subquery2:subq1-subquery2:x1] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src [null-subquery1:subq1-subquery1:x, null-subquery2:subq1-subquery2:x1] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -233,12 +233,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -249,12 +249,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src name: src @@ -266,14 +266,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -283,28 +283,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -315,12 +315,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -328,9 +328,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 [pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-43_964_5226296681156041746/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-19_150_396302363179050554/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -341,12 +341,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -357,12 +357,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205743 + transient_lastDdlTime 1288897939 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -398,11 +398,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-55-49_613_5191052733380652366/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-12-29_322_6593405909163555482/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-55-49_613_5191052733380652366/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-12-29_322_6593405909163555482/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.FieldSchema(name:value, type:string, comment:default), (src)x1.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/bucketmapjoin1.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin1.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin1.q.out (working copy) @@ -104,7 +104,7 @@ TableScan alias: b GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} {ds} @@ -117,10 +117,10 @@ Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[srcbucket20.txt, srcbucket22.txt], srcbucket21.txt=[srcbucket21.txt, srcbucket23.txt]} Alias Bucket File Name Mapping: - b {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} + b {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 Stage: Stage-1 Map Reduce @@ -169,9 +169,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -181,12 +181,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389533 + transient_lastDdlTime 1289266321 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -196,9 +196,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -210,12 +210,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389526 + transient_lastDdlTime 1289266314 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -227,12 +227,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389526 + transient_lastDdlTime 1289266314 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -244,14 +244,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -261,28 +261,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389533 + transient_lastDdlTime 1289266321 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -293,12 +293,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389533 + transient_lastDdlTime 1289266321 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -306,9 +306,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-58-53_791_6011037168447015801/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-32-01_093_6438431130513992258/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -319,12 +319,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389533 + transient_lastDdlTime 1289266321 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -335,12 +335,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389533 + transient_lastDdlTime 1289266321 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -368,11 +368,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-05_342_6895739373170494247/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-19_323_1265717156673930447/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-05_342_6895739373170494247/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-19_323_1265717156673930447/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part)b.FieldSchema(name:value, type:string, comment:null), ] @@ -421,11 +421,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-22_992_2572857688878915861/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-47_309_2016345755641040407/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-22_992_2572857688878915861/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-47_309_2016345755641040407/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -464,14 +464,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-30_539_1973835509487818054/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-58_179_7572353455658846533/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-30_539_1973835509487818054/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-32-58_179_7572353455658846533/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -533,7 +533,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} {ds} @@ -546,12 +546,12 @@ Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket20.txt], srcbucket23.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 Stage: Stage-1 Map Reduce @@ -600,9 +600,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -612,7 +612,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -621,7 +621,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288389562 + transient_lastDdlTime 1289266367 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -631,9 +631,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -647,13 +647,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389526 + transient_lastDdlTime 1289266314 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -665,13 +665,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389526 + transient_lastDdlTime 1289266314 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part name: srcbucket_mapjoin_part @@ -683,14 +683,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -700,7 +700,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -709,23 +709,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288389562 + transient_lastDdlTime 1289266367 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -736,7 +736,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -745,7 +745,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288389562 + transient_lastDdlTime 1289266367 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -753,9 +753,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-59-34_342_4747418016844297639/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-33-03_655_3362045352453175604/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -766,7 +766,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -775,7 +775,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288389562 + transient_lastDdlTime 1289266367 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -786,7 +786,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -795,7 +795,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288389562 + transient_lastDdlTime 1289266367 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -835,11 +835,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-47_962_3933759886396394149/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-33-23_257_4779027411421792236/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-59-47_962_3933759886396394149/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-33-23_257_4779027411421792236/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -924,11 +924,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-06_727_5057906343335838471/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-33-50_591_5983690384247835143/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-06_727_5057906343335838471/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-33-50_591_5983690384247835143/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] @@ -991,14 +991,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-15_088_8021410610659483984/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-34-03_738_8277069614113731460/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-15_088_8021410610659483984/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-34-03_738_8277069614113731460/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] Index: src/test/results/clientpositive/join26.q.out =================================================================== --- src/test/results/clientpositive/join26.q.out (revision 1032800) +++ src/test/results/clientpositive/join26.q.out (working copy) @@ -42,7 +42,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -57,7 +57,7 @@ TableScan alias: y GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -122,9 +122,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -134,12 +134,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391859 + transient_lastDdlTime 1289271563 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -149,9 +149,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat @@ -165,13 +165,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -182,13 +182,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart name: srcpart @@ -200,14 +200,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -217,28 +217,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391859 + transient_lastDdlTime 1289271563 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -249,12 +249,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391859 + transient_lastDdlTime 1289271563 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -262,9 +262,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-39_549_7180911715899470245/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-23_623_4518960640790293704/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -275,12 +275,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391859 + transient_lastDdlTime 1289271563 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -291,12 +291,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391859 + transient_lastDdlTime 1289271563 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -326,11 +326,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-37-46_064_2821265011132961568/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_18-59-33_893_3693465001557725052/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-37-46_064_2821265011132961568/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_18-59-33_893_3693465001557725052/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/bucketmapjoin_negative.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin_negative.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin_negative.q.out (working copy) @@ -74,7 +74,7 @@ TableScan alias: b GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} {ds} @@ -131,9 +131,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -143,12 +143,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390025 + transient_lastDdlTime 1289269076 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -158,9 +158,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -172,12 +172,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390021 + transient_lastDdlTime 1289269072 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -189,12 +189,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390021 + transient_lastDdlTime 1289269072 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -206,14 +206,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -223,28 +223,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390025 + transient_lastDdlTime 1289269076 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -255,12 +255,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390025 + transient_lastDdlTime 1289269076 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -268,9 +268,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-07-05_551_8783991555605079598/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-17-56_639_983307583097560485/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -281,12 +281,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390025 + transient_lastDdlTime 1289269076 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -297,12 +297,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288390025 + transient_lastDdlTime 1289269076 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result Index: src/test/results/clientpositive/skewjoin.q.out =================================================================== --- src/test/results/clientpositive/skewjoin.q.out (revision 1032800) +++ src/test/results/clientpositive/skewjoin.q.out (working copy) @@ -139,7 +139,7 @@ limit: -1 Alias -> Map Local Operator Tree: 1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {0_VALUE_0} 1 {1_VALUE_0} @@ -219,11 +219,11 @@ PREHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1 PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-19_102_8796456594111978797/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-02_177_6322462823933452249/-mr-10000 POSTHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-19_102_8796456594111978797/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-02_177_6322462823933452249/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 278697 101852390308 @@ -373,7 +373,7 @@ PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 PREHOOK: Input: default@t4 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-35_021_282318098948564636/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-48_767_6312998778554021237/-mr-10000 POSTHOOK: query: SELECT /*+ STREAMTABLE(a) */ * FROM T1 a JOIN T2 b ON a.key = b.key JOIN T3 c ON b.key = c.key @@ -383,7 +383,7 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 POSTHOOK: Input: default@t4 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-35_021_282318098948564636/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-48_767_6312998778554021237/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 2 12 2 22 2 12 2 12 @@ -533,7 +533,7 @@ PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 PREHOOK: Input: default@t4 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-38_270_4680889030113901410/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-55_262_4307015827448578323/-mr-10000 POSTHOOK: query: SELECT /*+ STREAMTABLE(a,c) */ * FROM T1 a JOIN T2 b ON a.key = b.key JOIN T3 c ON b.key = c.key @@ -543,7 +543,7 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 POSTHOOK: Input: default@t4 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-38_270_4680889030113901410/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-24-55_262_4307015827448578323/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 2 12 2 22 2 12 2 12 @@ -633,7 +633,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-06-41_304_411032267288143321/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-25-01_489_621733224236629264/-mr-10002 Reduce Output Operator sort order: tag: -1 @@ -678,12 +678,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-41_378_6980884094838374214/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-25-01_614_8452579245689090471/-mr-10000 POSTHOOK: query: FROM T1 a JOIN src c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-41_378_6980884094838374214/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-25-01_614_8452579245689090471/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 198 6274 194 @@ -800,7 +800,7 @@ limit: -1 Alias -> Map Local Operator Tree: 1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 1 {1_VALUE_0} {1_VALUE_1} @@ -852,7 +852,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-06-47_083_7583033798461542975/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-25-11_476_8171195744821612387/-mr-10002 Reduce Output Operator sort order: tag: -1 @@ -896,7 +896,7 @@ SELECT sum(hash(Y.key)), sum(hash(Y.value)) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-47_255_6693309924800490257/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-25-11_753_1076291109958736199/-mr-10000 POSTHOOK: query: FROM (SELECT src.* FROM src) x JOIN @@ -905,7 +905,7 @@ SELECT sum(hash(Y.key)), sum(hash(Y.value)) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-06-47_255_6693309924800490257/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-25-11_753_1076291109958736199/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 44481300 101852390308 @@ -1032,7 +1032,7 @@ limit: -1 Alias -> Map Local Operator Tree: 1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 1 {1_VALUE_0} {1_VALUE_1} @@ -1084,7 +1084,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-07-15_195_639976818225884535/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-26-43_698_1130450755234603131/-mr-10002 Reduce Output Operator sort order: tag: -1 @@ -1128,7 +1128,7 @@ SELECT sum(hash(Y.key)), sum(hash(Y.value)) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-07-15_370_1979544668421346379/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-26-43_990_6372868616213782508/-mr-10000 POSTHOOK: query: FROM (SELECT src.* FROM src) x JOIN @@ -1137,7 +1137,7 @@ SELECT sum(hash(Y.key)), sum(hash(Y.value)) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-07-15_370_1979544668421346379/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-26-43_990_6372868616213782508/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] NULL NULL @@ -1303,7 +1303,7 @@ limit: -1 Alias -> Map Local Operator Tree: 1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {0_VALUE_0} 1 {1_VALUE_0} @@ -1315,7 +1315,7 @@ 2 [Column[joinkey0]] Position of Big Table: 0 2 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {0_VALUE_0} 1 {1_VALUE_0} @@ -1372,7 +1372,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-07-40_939_791224492830751906/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-28-07_199_3806389747749479816/-mr-10002 Reduce Output Operator sort order: tag: -1 @@ -1414,7 +1414,7 @@ limit: -1 Alias -> Map Local Operator Tree: 0 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {0_VALUE_0} 1 {1_VALUE_0} @@ -1426,7 +1426,7 @@ 2 [Column[joinkey0]] Position of Big Table: 1 2 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {0_VALUE_0} 1 {1_VALUE_0} @@ -1496,7 +1496,7 @@ ON src1.c1 = src3.c5 AND src3.c5 < 80 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-07-41_289_4528597985146531444/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-28-07_762_745139769279053433/-mr-10000 POSTHOOK: query: SELECT sum(hash(src1.c1)), sum(hash(src2.c4)) FROM (SELECT src.key as c1, src.value as c2 from src) src1 @@ -1508,7 +1508,7 @@ ON src1.c1 = src3.c5 AND src3.c5 < 80 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-07-41_289_4528597985146531444/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-28-07_762_745139769279053433/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 293143 -136853010385 @@ -1540,7 +1540,7 @@ v TableScan alias: v - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {val} @@ -1580,7 +1580,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-08-11_023_4952961378451488185/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-30-14_923_356309771096230331/-mr-10002 Select Operator expressions: expr: _col0 @@ -1640,55 +1640,55 @@ PREHOOK: query: SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-11_093_7547555083303240518/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-15_048_6024162501211427411/-mr-10000 POSTHOOK: query: SELECT /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) FROM T1 k LEFT OUTER JOIN T1 v ON k.key+1=v.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-11_093_7547555083303240518/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-15_048_6024162501211427411/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 372 6320 PREHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-16_379_6728265708807350086/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-26_961_8307603098035847510/-mr-10000 POSTHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.val POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-16_379_6728265708807350086/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-26_961_8307603098035847510/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] NULL NULL PREHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-21_607_6541620241735835708/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-36_802_3005209775052152224/-mr-10000 POSTHOOK: query: select /*+ mapjoin(k)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-21_607_6541620241735835708/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-36_802_3005209775052152224/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 429 12643 PREHOOK: query: select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-27_482_5011839329231174267/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-48_466_8583634665917897680/-mr-10000 POSTHOOK: query: select sum(hash(k.key)), sum(hash(v.val)) from T1 k join T1 v on k.key=v.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-27_482_5011839329231174267/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-30-48_466_8583634665917897680/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 429 12643 PREHOOK: query: select count(1) from T1 a join T1 b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-35_146_3336063612709764146/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-03_237_192983720880547949/-mr-10000 POSTHOOK: query: select count(1) from T1 a join T1 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-35_146_3336063612709764146/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-03_237_192983720880547949/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 8 @@ -1696,12 +1696,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-42_859_1470660249312294617/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-16_676_2618061858875139981/-mr-10000 POSTHOOK: query: FROM T1 a LEFT OUTER JOIN T2 c ON c.key+1=a.key SELECT sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-42_859_1470660249312294617/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-16_676_2618061858875139981/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 317 9462 50 @@ -1709,12 +1709,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-48_569_1389223424240589811/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-25_804_1434715960848718635/-mr-10000 POSTHOOK: query: FROM T1 a RIGHT OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-48_569_1389223424240589811/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-25_804_1434715960848718635/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 51 1570 318 @@ -1722,12 +1722,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-53_884_193002294179789118/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-35_151_4513393491876936064/-mr-10000 POSTHOOK: query: FROM T1 a FULL OUTER JOIN T2 c ON c.key+1=a.key SELECT /*+ STREAMTABLE(a) */ sum(hash(a.key)), sum(hash(a.val)), sum(hash(c.key)) POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-53_884_193002294179789118/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-35_151_4513393491876936064/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 317 9462 318 @@ -1735,12 +1735,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-59_493_4647924292571801277/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-44_380_3162963056826185972/-mr-10000 POSTHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 LEFT OUTER JOIN T2 src2 ON src1.key+1 = src2.key RIGHT OUTER JOIN T2 src3 ON src2.key = src3.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-08-59_493_4647924292571801277/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-44_380_3162963056826185972/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 370 11003 377 @@ -1748,23 +1748,23 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-09-04_974_5598263675862728012/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-53_546_8829290719525806208/-mr-10000 POSTHOOK: query: SELECT sum(hash(src1.key)), sum(hash(src1.val)), sum(hash(src2.key)) FROM T1 src1 JOIN T2 src2 ON src1.key+1 = src2.key JOIN T2 src3 ON src2.key = src3.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-09-04_974_5598263675862728012/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-31-53_546_8829290719525806208/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 370 11003 377 PREHOOK: query: select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-09-15_753_4549651274151249293/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-32-19_241_1942705791648142414/-mr-10000 POSTHOOK: query: select /*+ mapjoin(v)*/ sum(hash(k.key)), sum(hash(v.val)) from T1 k left outer join T1 v on k.key+1=v.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-09-15_753_4549651274151249293/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-32-19_241_1942705791648142414/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] 372 6320 Index: src/test/results/clientpositive/join35.q.out =================================================================== --- src/test/results/clientpositive/join35.q.out (revision 1032800) +++ src/test/results/clientpositive/join35.q.out (working copy) @@ -83,9 +83,9 @@ type: bigint Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src [null-subquery1:subq1-subquery1:x] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src [null-subquery1:subq1-subquery1:x] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -96,12 +96,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -112,12 +112,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src name: src @@ -141,7 +141,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10002 + directory: file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10002 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -165,7 +165,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col1} 1 {key} {value} @@ -178,7 +178,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10002 Union Map Join Operator condition map: @@ -222,9 +222,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -234,18 +234,18 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 GatherStats: true MultiFileSpray: false - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10004 + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10004 Union Map Join Operator condition map: @@ -289,9 +289,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -301,12 +301,12 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -316,10 +316,10 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10002 [file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10002] - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10004 [file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10004] + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10002 [file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10002] + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10004 [file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10004] Path -> Partition: - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10002 Partition base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -335,7 +335,7 @@ columns _col0,_col1 columns.types string,bigint escape.delim \ - file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10004 + file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10004 Partition base file name: -mr-10004 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -359,14 +359,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -376,28 +376,28 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10001 Stage: Stage-3 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000/ Stage: Stage-4 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -408,12 +408,12 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -421,9 +421,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 [pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-55-54_179_6477900064401201039/-ext-10003 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-12-33_932_3206165386907828885/-ext-10003 Partition base file name: -ext-10003 input format: org.apache.hadoop.mapred.TextInputFormat @@ -434,12 +434,12 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -450,12 +450,12 @@ columns.types string:string:int file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, i32 val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205754 + transient_lastDdlTime 1288897953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -505,9 +505,9 @@ type: bigint Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src [null-subquery2:subq1-subquery2:x1] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src [null-subquery2:subq1-subquery2:x1] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -518,12 +518,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -534,12 +534,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205427 + transient_lastDdlTime 1288894219 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src name: src @@ -563,7 +563,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/njain/hive_2010-10-27_11-55-54_179_6477900064401201039/-mr-10004 + directory: file:/tmp/liyintang/hive_2010-11-04_12-12-33_932_3206165386907828885/-mr-10004 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -607,11 +607,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-06_687_1264215794989002476/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-12-57_316_845546887873454795/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-06_687_1264215794989002476/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-12-57_316_845546887873454795/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/bucketmapjoin2.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin2.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin2.q.out (working copy) @@ -109,7 +109,7 @@ predicate: expr: (ds = '2008-04-08') type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -122,10 +122,10 @@ Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[srcbucket22.txt], srcbucket21.txt=[srcbucket23.txt]} Alias Bucket File Name Mapping: - b {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt]} + b {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 Stage: Stage-1 Map Reduce @@ -167,9 +167,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -179,12 +179,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389631 + transient_lastDdlTime 1289266460 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -194,9 +194,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -208,12 +208,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389622 + transient_lastDdlTime 1289266453 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -225,12 +225,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389622 + transient_lastDdlTime 1289266453 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -242,14 +242,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -259,28 +259,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389631 + transient_lastDdlTime 1289266460 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -291,12 +291,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389631 + transient_lastDdlTime 1289266460 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -304,9 +304,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-00-31_195_289123554843212407/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-34-20_832_2901319784199272686/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -317,12 +317,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389631 + transient_lastDdlTime 1289266460 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -333,12 +333,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389631 + transient_lastDdlTime 1289266460 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -366,11 +366,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-43_902_7002747651117209826/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-34-36_359_6171516716162381131/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-00-43_902_7002747651117209826/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-34-36_359_6171516716162381131/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part_2)b.FieldSchema(name:value, type:string, comment:null), ] @@ -419,11 +419,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-01_443_2769919476244969772/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-01_891_630655332694050726/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-01_443_2769919476244969772/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-01_891_630655332694050726/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -462,14 +462,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-09_555_4586184703837197207/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-13_985_5479572132827959046/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-09_555_4586184703837197207/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-13_985_5479572132827959046/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -531,7 +531,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -544,10 +544,10 @@ Alias Bucket Base File Name Mapping: a {srcbucket22.txt=[srcbucket20.txt], srcbucket23.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 Stage: Stage-1 Map Reduce @@ -594,9 +594,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -606,7 +606,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -615,7 +615,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 0 - transient_lastDdlTime 1288389661 + transient_lastDdlTime 1289266501 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -625,9 +625,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [b] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -641,13 +641,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389628 + transient_lastDdlTime 1289266458 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -659,13 +659,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389628 + transient_lastDdlTime 1289266458 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part_2 name: srcbucket_mapjoin_part_2 @@ -677,14 +677,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -694,7 +694,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -703,23 +703,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 0 - transient_lastDdlTime 1288389661 + transient_lastDdlTime 1289266501 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -730,7 +730,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -739,7 +739,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 0 - transient_lastDdlTime 1288389661 + transient_lastDdlTime 1289266501 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -747,9 +747,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-01-13_322_8516005593892626284/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-35-19_461_1296140382333824913/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -760,7 +760,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -769,7 +769,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 0 - transient_lastDdlTime 1288389661 + transient_lastDdlTime 1289266501 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -780,7 +780,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -789,7 +789,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 0 - transient_lastDdlTime 1288389661 + transient_lastDdlTime 1289266501 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -829,11 +829,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-23_700_2459916685803827159/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-35_044_8009627533587099101/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-23_700_2459916685803827159/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-35-35_044_8009627533587099101/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -918,11 +918,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-40_801_6422166850549493252/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-01_094_6860647864472235592/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-40_801_6422166850549493252/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-01_094_6860647864472235592/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] @@ -985,14 +985,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-48_532_707937022506405457/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-12_221_3121293415450965317/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-01-48_532_707937022506405457/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-12_221_3121293415450965317/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] Index: src/test/results/clientpositive/join_map_ppr.q.out =================================================================== --- src/test/results/clientpositive/join_map_ppr.q.out (revision 1032800) +++ src/test/results/clientpositive/join_map_ppr.q.out (working copy) @@ -44,7 +44,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -59,7 +59,7 @@ TableScan alias: y GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -123,9 +123,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -135,12 +135,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288392639 + transient_lastDdlTime 1289272690 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -150,9 +150,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat @@ -166,13 +166,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -183,13 +183,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart name: srcpart @@ -201,14 +201,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -218,28 +218,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288392639 + transient_lastDdlTime 1289272690 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -250,12 +250,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288392639 + transient_lastDdlTime 1289272690 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -263,9 +263,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-39_603_533413961679906692/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-10_467_2293155916770670069/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -276,12 +276,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288392639 + transient_lastDdlTime 1289272690 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -292,12 +292,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288392639 + transient_lastDdlTime 1289272690 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -329,11 +329,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-50-46_099_4098924361433737950/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-18-20_736_2131780804835036352/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-50-46_099_4098924361433737950/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-18-20_736_2131780804835036352/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] @@ -536,7 +536,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -551,7 +551,7 @@ TableScan alias: y GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 {value} @@ -615,9 +615,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -627,7 +627,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 numFiles 1 numPartitions 0 @@ -636,7 +636,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 2125 - transient_lastDdlTime 1288392646 + transient_lastDdlTime 1289272700 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -646,9 +646,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [z] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat @@ -662,13 +662,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -679,13 +679,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcpart + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcpart name srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389287 + transient_lastDdlTime 1289268000 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart name: srcpart @@ -697,14 +697,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -714,7 +714,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 numFiles 1 numPartitions 0 @@ -723,23 +723,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 2125 - transient_lastDdlTime 1288392646 + transient_lastDdlTime 1289272700 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -750,7 +750,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 numFiles 1 numPartitions 0 @@ -759,7 +759,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 2125 - transient_lastDdlTime 1288392646 + transient_lastDdlTime 1289272700 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -767,9 +767,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-50-58_503_3505008876681402750/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-18-37_907_7726137335023848601/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -780,7 +780,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 numFiles 1 numPartitions 0 @@ -789,7 +789,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 2125 - transient_lastDdlTime 1288392646 + transient_lastDdlTime 1289272700 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -800,7 +800,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 numFiles 1 numPartitions 0 @@ -809,7 +809,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 2125 - transient_lastDdlTime 1288392646 + transient_lastDdlTime 1289272700 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -848,11 +848,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-51-06_013_6248158911838781364/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-18-49_147_7316712558387738462/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-51-06_013_6248158911838781364/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-18-49_147_7316712558387738462/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1_copy)x.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/join30.q.out =================================================================== --- src/test/results/clientpositive/join30.q.out (revision 1032800) +++ src/test/results/clientpositive/join30.q.out (working copy) @@ -32,7 +32,7 @@ x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -72,7 +72,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-54-49_452_8944509394940573381/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-10-56_018_3645639935399412130/-mr-10002 Select Operator expressions: expr: _col0 @@ -168,11 +168,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-56_890_5863714555389240048/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-11-08_869_436561712847189276/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-56_890_5863714555389240048/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-11-08_869_436561712847189276/-mr-10000 POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] 66 1 Index: src/test/results/clientpositive/semijoin.q.out =================================================================== --- src/test/results/clientpositive/semijoin.q.out (revision 1032800) +++ src/test/results/clientpositive/semijoin.q.out (working copy) @@ -8,11 +8,11 @@ PREHOOK: query: select * from t1 sort by key PREHOOK: type: QUERY PREHOOK: Input: default@t1 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-07_317_2567121123918340747/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-35_964_8659428901868696782/-mr-10000 POSTHOOK: query: select * from t1 sort by key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-07_317_2567121123918340747/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-35_964_8659428901868696782/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -34,11 +34,11 @@ PREHOOK: query: select * from t2 sort by key PREHOOK: type: QUERY PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-12_609_8186818336637614021/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-44_033_1385561037107185571/-mr-10000 POSTHOOK: query: select * from t2 sort by key POSTHOOK: type: QUERY POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-12_609_8186818336637614021/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-44_033_1385561037107185571/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -62,11 +62,11 @@ PREHOOK: query: select * from t3 sort by key, value PREHOOK: type: QUERY PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-20_412_6858946112655112787/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-56_368_4643337669577300642/-mr-10000 POSTHOOK: query: select * from t3 sort by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-20_412_6858946112655112787/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-56_368_4643337669577300642/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -97,11 +97,11 @@ PREHOOK: query: select * from t4 PREHOOK: type: QUERY PREHOOK: Input: default@t4 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-23_040_6896567563297871765/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-00_347_4261760726152540894/-mr-10000 POSTHOOK: query: select * from t4 POSTHOOK: type: QUERY POSTHOOK: Input: default@t4 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-23_040_6896567563297871765/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-00_347_4261760726152540894/-mr-10000 PREHOOK: query: explain select * from t1 a left semi join t2 b on a.key=b.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t1 a left semi join t2 b on a.key=b.key sort by a.key, a.value @@ -185,7 +185,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-23_122_1418801017341736948/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-00_543_3830170995670719849/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -217,12 +217,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-23_188_6557146902672307691/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-00_658_5539359917304769093/-mr-10000 POSTHOOK: query: select * from t1 a left semi join t2 b on a.key=b.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-23_188_6557146902672307691/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-00_658_5539359917304769093/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -312,7 +312,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-28_893_6723532886712811629/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-08_284_4216408335407237139/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -344,12 +344,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-28_959_3762364132590482286/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-08_398_3149624530689509743/-mr-10000 POSTHOOK: query: select * from t2 a left semi join t1 b on b.key=a.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-28_959_3762364132590482286/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-08_398_3149624530689509743/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -441,7 +441,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-34_570_7889009522221032190/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-16_135_6319067187632899075/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -473,12 +473,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t4 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-34_654_5556420496433395304/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-16_255_6778253691436748699/-mr-10000 POSTHOOK: query: select * from t1 a left semi join t4 b on b.key=a.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t4 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-34_654_5556420496433395304/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-16_255_6778253691436748699/-mr-10000 PREHOOK: query: explain select a.value from t1 a left semi join t3 b on (b.key = a.key and b.key < '15') sort by a.value PREHOOK: type: QUERY POSTHOOK: query: explain select a.value from t1 a left semi join t3 b on (b.key = a.key and b.key < '15') sort by a.value @@ -568,7 +568,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-40_011_3808833152580249248/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-24_857_4689023765690259253/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -596,12 +596,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-40_079_1282941878916492070/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-24_973_3041071756227440132/-mr-10000 POSTHOOK: query: select a.value from t1 a left semi join t3 b on (b.key = a.key and b.key < '15') sort by a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-40_079_1282941878916492070/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-24_973_3041071756227440132/-mr-10000 val_0 val_0 val_0 @@ -708,7 +708,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-45_730_7298072930043033983/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-34_061_8275163351633601557/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -740,12 +740,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-45_804_5294995386951265093/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-34_179_4205035139034381615/-mr-10000 POSTHOOK: query: select * from t1 a left semi join t2 b on a.key = b.key and b.value < "val_10" sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-45_804_5294995386951265093/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-34_179_4205035139034381615/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -841,7 +841,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-51_407_427333938274086877/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-43_183_3107199938637789566/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -869,12 +869,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-51_486_1315085131198914442/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-43_306_2053779749324913446/-mr-10000 POSTHOOK: query: select a.value from t1 a left semi join (select key from t3 where key > 5) b on a.key = b.key sort by a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-51_486_1315085131198914442/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-43_306_2053779749324913446/-mr-10000 val_10 val_8 val_9 @@ -980,7 +980,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-03-57_031_4644400431266147309/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-18-52_335_849441347171062507/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1008,12 +1008,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-57_105_2695825645438377544/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-52_463_737069354382102340/-mr-10000 POSTHOOK: query: select a.value from t1 a left semi join (select key , value from t2 where key > 5) b on a.key = b.key and b.value <= 'val_20' sort by a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-57_105_2695825645438377544/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-18-52_463_737069354382102340/-mr-10000 PREHOOK: query: explain select * from t2 a left semi join (select key , value from t1 where key > 2) b on a.key = b.key sort by a.key, a.value PREHOOK: type: QUERY POSTHOOK: query: explain select * from t2 a left semi join (select key , value from t1 where key > 2) b on a.key = b.key sort by a.key, a.value @@ -1110,7 +1110,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-02_466_725728290759835787/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-00_301_5958832649298868830/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1142,12 +1142,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-02_537_8369733971186993079/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-00_424_4463918460829143379/-mr-10000 POSTHOOK: query: select * from t2 a left semi join (select key , value from t1 where key > 2) b on a.key = b.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-02_537_8369733971186993079/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-00_424_4463918460829143379/-mr-10000 4 val_2 8 val_4 10 val_5 @@ -1189,7 +1189,7 @@ type: int mode: hash outputColumnNames: _col0 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -1229,7 +1229,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-08_036_2910952596121389668/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-09_365_1316122738460523098/-mr-10002 Select Operator expressions: expr: _col0 @@ -1267,12 +1267,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-08_109_4473865912549348136/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-09_480_7870074811327558689/-mr-10000 POSTHOOK: query: select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-08_109_4473865912549348136/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-09_480_7870074811327558689/-mr-10000 0 0 0 @@ -1375,7 +1375,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-13_353_2384479425001200067/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-19_175_5090290266309302820/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1407,12 +1407,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-13_420_5777699289863486696/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-19_289_7283573667957611898/-mr-10000 POSTHOOK: query: select * from t1 a left semi join t2 b on a.key = 2*b.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-13_420_5777699289863486696/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-19_289_7283573667957611898/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -1523,7 +1523,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-18_912_2891401415653997089/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-28_320_7044231815158137622/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1560,13 +1560,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-18_988_5010287233526168835/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-28_450_6078674943204864805/-mr-10000 POSTHOOK: query: select * from t1 a join t2 b on a.key = b.key left semi join t3 c on b.key = c.key sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-18_988_5010287233526168835/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-28_450_6078674943204864805/-mr-10000 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 @@ -1676,7 +1676,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-24_617_6131522087162404256/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-38_052_8721634945636524946/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1708,12 +1708,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-24_685_7575816128654858988/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-38_174_5762672611483777652/-mr-10000 POSTHOOK: query: select * from t3 a left semi join t1 b on a.key = b.key and a.value=b.value sort by a.key, a.value POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-24_685_7575816128654858988/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-38_174_5762672611483777652/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -1767,7 +1767,7 @@ type: int mode: hash outputColumnNames: _col0 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -1793,7 +1793,7 @@ type: int mode: hash outputColumnNames: _col0 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -1838,7 +1838,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-30_183_6165676837426268350/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-49_190_4624457787442565360/-mr-10002 Select Operator expressions: expr: _col0 @@ -1877,13 +1877,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-30_260_2433281365180402598/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-49_324_9174162734156458916/-mr-10000 POSTHOOK: query: select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-30_260_2433281365180402598/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-49_324_9174162734156458916/-mr-10000 0 0 0 @@ -1991,7 +1991,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-35_797_2333906634777100148/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-19-59_535_8715349469270136515/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2020,13 +2020,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-35_871_7897675469131546389/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-59_667_8557238197192129755/-mr-10000 POSTHOOK: query: select a.key from t3 a left outer join t1 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-35_871_7897675469131546389/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-19-59_667_8557238197192129755/-mr-10000 0 0 0 @@ -2146,7 +2146,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-41_608_2220388568134618866/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-20-09_255_2715293241143785335/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2175,13 +2175,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-41_682_2882263810051097750/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-09_383_4275977555046345305/-mr-10000 POSTHOOK: query: select a.key from t1 a right outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-41_682_2882263810051097750/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-09_383_4275977555046345305/-mr-10000 NULL NULL NULL @@ -2304,7 +2304,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-48_006_8185934732491634852/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-20-19_268_5618148586891473846/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2333,13 +2333,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-48_081_2028444012866190155/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-19_400_3587882397101268241/-mr-10000 POSTHOOK: query: select a.key from t1 a full outer join t3 b on a.key = b.key left semi join t2 c on b.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-48_081_2028444012866190155/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-19_400_3587882397101268241/-mr-10000 NULL NULL NULL @@ -2462,7 +2462,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-53_653_9010857136935696124/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-20-28_728_1492881717279570087/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2491,13 +2491,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-53_728_7613723015009146650/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-28_856_8757358207361872372/-mr-10000 POSTHOOK: query: select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-53_728_7613723015009146650/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-28_856_8757358207361872372/-mr-10000 0 0 0 @@ -2620,7 +2620,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-04-59_367_1281949091297790008/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-20-38_076_4685193258029073996/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2649,13 +2649,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-59_442_8906799826368408100/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-38_205_3988929412565665845/-mr-10000 POSTHOOK: query: select a.key from t3 a left semi join t2 b on a.key = b.key right outer join t1 c on a.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-04-59_442_8906799826368408100/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-38_205_3988929412565665845/-mr-10000 NULL NULL NULL @@ -2780,7 +2780,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-05-08_027_9057656913862672973/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_13-20-47_590_2999113315834306029/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2809,13 +2809,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-05-08_102_1904204088263319648/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-47_722_4677693091558572786/-mr-10000 POSTHOOK: query: select a.key from t3 a left semi join t1 b on a.key = b.key full outer join t2 c on a.key = c.key sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-05-08_102_1904204088263319648/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-47_722_4677693091558572786/-mr-10000 NULL NULL NULL @@ -2984,7 +2984,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - file:/tmp/heyongqiang/hive_2010-10-27_15-05-13_732_1908006251522430032/-mr-10003 + file:/tmp/liyintang/hive_2010-11-04_13-20-57_841_8595738346509621581/-mr-10003 Reduce Output Operator key expressions: expr: _col0 @@ -3013,13 +3013,13 @@ PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-05-13_810_2454226698258102980/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-57_976_1019112090965524872/-mr-10000 POSTHOOK: query: select a.key from t3 a left semi join t2 b on a.key = b.key left outer join t1 c on a.value = c.value sort by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-05-13_810_2454226698258102980/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-20-57_976_1019112090965524872/-mr-10000 0 0 0 Index: src/test/results/clientpositive/join27.q.out =================================================================== --- src/test/results/clientpositive/join27.q.out (revision 1032800) +++ src/test/results/clientpositive/join27.q.out (working copy) @@ -36,7 +36,7 @@ x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -109,7 +109,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-54-17_054_2541334443319399043/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-09-55_355_6415074899212151208/-ext-10000 Stage: Stage-0 Move Operator @@ -127,7 +127,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-54-17_054_2541334443319399043/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-09-55_355_6415074899212151208/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -158,11 +158,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key, x.value PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-22_385_7137802075538214766/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-10-05_344_6343916336537575010/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key, x.value POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-54-22_385_7137802075538214766/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-10-05_344_6343916336537575010/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/mapjoin_subquery.q.out =================================================================== --- src/test/results/clientpositive/mapjoin_subquery.q.out (revision 1032800) +++ src/test/results/clientpositive/mapjoin_subquery.q.out (working copy) @@ -34,7 +34,7 @@ subq:x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -54,7 +54,7 @@ predicate: expr: (hr = 11) type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col0} 1 {value} @@ -141,7 +141,7 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-07-30_120_6146413041257039963/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-40-13_880_6177591846608438614/-mr-10000 POSTHOOK: query: SELECT /*+ MAPJOIN(z) */ subq.key1, z.value FROM (SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2 @@ -151,7 +151,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-07-30_120_6146413041257039963/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-40-13_880_6177591846608438614/-mr-10000 238 val_238 238 val_238 311 val_311 @@ -298,7 +298,7 @@ subq:x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -318,7 +318,7 @@ predicate: expr: (hr = 11) type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col0} 1 {value} @@ -380,7 +380,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2010-10-29_16-07-34_857_8300003237112409468/-mr-10002 + file:/tmp/liyintang/hive_2010-11-08_19-40-20_684_3855178470484320112/-mr-10002 Select Operator expressions: expr: _col0 @@ -430,7 +430,7 @@ PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-07-35_005_4332334219474823384/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-40-20_868_280957277267588781/-mr-10000 POSTHOOK: query: SELECT /*+ MAPJOIN(z) */ subq.key1, z.value FROM (SELECT /*+ MAPJOIN(x) */ x.key as key1, x.value as value1, y.key as key2, y.value as value2 @@ -441,7 +441,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-07-35_005_4332334219474823384/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-40-20_868_280957277267588781/-mr-10000 128 val_128 128 val_128 128 val_128 Index: src/test/results/clientpositive/join36.q.out =================================================================== --- src/test/results/clientpositive/join36.q.out (revision 1032800) +++ src/test/results/clientpositive/join36.q.out (working copy) @@ -76,7 +76,7 @@ x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {cnt} 1 {cnt} @@ -140,7 +140,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-56-20_127_6380703738706956749/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-13-17_369_4823166777089074004/-ext-10000 Stage: Stage-0 Move Operator @@ -158,7 +158,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-56-20_127_6380703738706956749/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-13-17_369_4823166777089074004/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -193,11 +193,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-26_754_2259038051216019843/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-27_195_6659028372004858698/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-26_754_2259038051216019843/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-27_195_6659028372004858698/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(tmp1)x.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(tmp2)y.FieldSchema(name:cnt, type:int, comment:null), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(tmp1)x.FieldSchema(name:cnt, type:int, comment:null), ] Index: src/test/results/clientpositive/bucketmapjoin3.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin3.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin3.q.out (working copy) @@ -109,7 +109,7 @@ predicate: expr: (ds = '2008-04-08') type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -122,10 +122,10 @@ Alias Bucket Base File Name Mapping: b {srcbucket22.txt=[srcbucket20.txt, srcbucket22.txt], srcbucket23.txt=[srcbucket21.txt, srcbucket23.txt]} Alias Bucket File Name Mapping: - b {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} + b {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt 1 Stage: Stage-1 Map Reduce @@ -172,9 +172,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -184,12 +184,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388787 + transient_lastDdlTime 1289266588 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -199,9 +199,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 [a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -215,13 +215,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388786 + transient_lastDdlTime 1289266586 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -233,13 +233,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2 name srcbucket_mapjoin_part_2 partition_columns ds serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388786 + transient_lastDdlTime 1289266586 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part_2 name: srcbucket_mapjoin_part_2 @@ -251,14 +251,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -268,28 +268,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388787 + transient_lastDdlTime 1289266588 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -300,12 +300,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388787 + transient_lastDdlTime 1289266588 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -313,9 +313,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-46-27_822_6794024936564446840/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-36-28_593_5656522145364934245/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -326,12 +326,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388787 + transient_lastDdlTime 1289266588 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -342,12 +342,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388787 + transient_lastDdlTime 1289266588 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -375,11 +375,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-46-39_730_5807364860703751088/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-45_045_5247628416990409526/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-46-39_730_5807364860703751088/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-36-45_045_5247628416990409526/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin_part_2)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin_part_2)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part)b.FieldSchema(name:value, type:string, comment:null), ] @@ -428,11 +428,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-46-57_859_5496400832390508341/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-11_355_4701836415203720955/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-46-57_859_5496400832390508341/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-11_355_4701836415203720955/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -471,14 +471,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-05_432_4411871243550250664/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-22_340_1809276735979281182/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-05_432_4411871243550250664/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-22_340_1809276735979281182/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -545,7 +545,7 @@ predicate: expr: (ds = '2008-04-08') type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -558,12 +558,12 @@ Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket22.txt], srcbucket21.txt=[srcbucket23.txt], srcbucket22.txt=[srcbucket22.txt], srcbucket23.txt=[srcbucket23.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part_2/ds=2008-04-08/srcbucket23.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 Stage: Stage-1 Map Reduce @@ -610,9 +610,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -622,7 +622,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -631,7 +631,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11067 - transient_lastDdlTime 1288388817 + transient_lastDdlTime 1289266631 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -641,9 +641,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -657,13 +657,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388783 + transient_lastDdlTime 1289266581 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -675,13 +675,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288388783 + transient_lastDdlTime 1289266581 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part name: srcbucket_mapjoin_part @@ -693,14 +693,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -710,7 +710,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -719,23 +719,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11067 - transient_lastDdlTime 1288388817 + transient_lastDdlTime 1289266631 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -746,7 +746,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -755,7 +755,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11067 - transient_lastDdlTime 1288388817 + transient_lastDdlTime 1289266631 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -763,9 +763,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_14-47-09_011_174641296277649851/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_17-37-28_134_766191094581042721/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -776,7 +776,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -785,7 +785,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11067 - transient_lastDdlTime 1288388817 + transient_lastDdlTime 1289266631 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -796,7 +796,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -805,7 +805,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11067 - transient_lastDdlTime 1288388817 + transient_lastDdlTime 1289266631 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -845,11 +845,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-21_496_5610263740296235510/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-46_601_4275896744676736010/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-21_496_5610263740296235510/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-37-46_601_4275896744676736010/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -934,11 +934,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-39_276_5483603111030632960/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-38-16_563_2631614957089053546/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-39_276_5483603111030632960/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-38-16_563_2631614957089053546/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -1001,14 +1001,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-46_750_8346891533793956132/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-38-27_870_6358329541404712380/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_14-47-46_750_8346891533793956132/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_17-38-27_870_6358329541404712380/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] Index: src/test/results/clientpositive/join31.q.out =================================================================== --- src/test/results/clientpositive/join31.q.out (revision 1032800) +++ src/test/results/clientpositive/join31.q.out (working copy) @@ -87,12 +87,12 @@ Stage: Stage-7 Map Reduce Local Work Alias -> Map Local Tables: - file:/tmp/njain/hive_2010-10-27_11-55-00_152_4091108333306141197/-mr-10004 + file:/tmp/liyintang/hive_2010-11-04_12-11-13_590_7577887823222105055/-mr-10004 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-55-00_152_4091108333306141197/-mr-10004 - JDBM Sink Operator + file:/tmp/liyintang/hive_2010-11-04_12-11-13_590_7577887823222105055/-mr-10004 + HashTable Sink Operator condition expressions: 0 {_col0} 1 @@ -105,7 +105,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-55-00_152_4091108333306141197/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-11-13_590_7577887823222105055/-mr-10002 Map Join Operator condition map: Inner Join 0 to 1 @@ -130,7 +130,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-55-00_152_4091108333306141197/-mr-10003 + file:/tmp/liyintang/hive_2010-11-04_12-11-13_590_7577887823222105055/-mr-10003 Select Operator expressions: expr: _col0 @@ -286,11 +286,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-55-14_996_8245169574800468911/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-11-35_085_715748887928903684/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-55-14_996_8245169574800468911/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-11-35_085_715748887928903684/-mr-10000 POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ] POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] 128 1 Index: src/test/results/clientpositive/join40.q.out =================================================================== --- src/test/results/clientpositive/join40.q.out (revision 1032800) +++ src/test/results/clientpositive/join40.q.out (working copy) @@ -100,12 +100,12 @@ FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-27_869_8594307870019576114/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-28_326_3972885048921144262/-mr-10000 POSTHOOK: query: SELECT x.key, x.value, y.key, y.value FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-27_869_8594307870019576114/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-28_326_3972885048921144262/-mr-10000 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 @@ -751,12 +751,12 @@ FROM src src1 JOIN src src2 ON (src1.key = src2.key) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-32_224_2628641681553791073/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-36_619_6415824732118440282/-mr-10000 POSTHOOK: query: select src1.key, src2.value FROM src src1 JOIN src src2 ON (src1.key = src2.key) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-32_224_2628641681553791073/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-36_619_6415824732118440282/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -1904,7 +1904,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-58-37_240_1047427345351102102/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-14-46_832_624292527665996520/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -1952,12 +1952,12 @@ SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-37_348_2776356054970751573/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-46_969_1334858435022180797/-mr-10000 POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-37_348_2776356054970751573/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-14-46_969_1334858435022180797/-mr-10000 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -2633,7 +2633,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-58-44_576_3870847117030797031/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-15-01_848_4481484445896033624/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -2681,12 +2681,12 @@ SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-44_694_461427396890734084/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-02_041_1716373562559394449/-mr-10000 POSTHOOK: query: SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-44_694_461427396890734084/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-02_041_1716373562559394449/-mr-10000 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 100 val_100 NULL NULL NULL NULL 100 val_100 @@ -3277,7 +3277,7 @@ expr: value type: string outputColumnNames: _col0, _col1 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {_col0} {_col1} @@ -3345,12 +3345,12 @@ FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-51_016_2800726535361834663/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-12_928_5277225703166373723/-mr-10000 POSTHOOK: query: SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value FROM src x left outer JOIN (select * from src where key <= 100) y ON (x.key = y.key) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-51_016_2800726535361834663/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-12_928_5277225703166373723/-mr-10000 238 val_238 NULL NULL 86 val_86 86 val_86 311 val_311 NULL NULL @@ -3984,7 +3984,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/njain/hive_2010-10-27_11-58-54_107_7492093354536600513/-mr-10002 + file:/tmp/liyintang/hive_2010-11-04_12-15-20_303_498904788302590605/-mr-10002 Reduce Output Operator sort order: tag: -1 @@ -4018,9 +4018,9 @@ PREHOOK: query: SELECT COUNT(1) FROM SRC A JOIN SRC B ON (A.KEY=B.KEY) PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-54_187_249714887699251169/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-20_419_6110829774550991242/-mr-10000 POSTHOOK: query: SELECT COUNT(1) FROM SRC A JOIN SRC B ON (A.KEY=B.KEY) POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-58-54_187_249714887699251169/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-15-20_419_6110829774550991242/-mr-10000 1028 Index: src/test/results/clientpositive/select_transform_hint.q.out =================================================================== --- src/test/results/clientpositive/select_transform_hint.q.out (revision 1032800) +++ src/test/results/clientpositive/select_transform_hint.q.out (working copy) @@ -29,7 +29,7 @@ a TableScan alias: a - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 @@ -96,14 +96,14 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-02-57_632_2812838087751245658/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-19_687_5387495731767365689/-mr-10000 POSTHOOK: query: SELECT /*+MAPJOIN(a)*/ TRANSFORM(a.key, a.value) USING '/bin/cat' AS (tkey, tvalue) FROM src a join src b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-02-57_632_2812838087751245658/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-19_687_5387495731767365689/-mr-10000 238 val_238 238 val_238 86 val_86 @@ -1223,14 +1223,14 @@ on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-01_031_3277101842984747988/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-27_114_2192122567440751595/-mr-10000 POSTHOOK: query: SELECT /*+STREAMTABLE(a)*/ TRANSFORM(a.key, a.value) USING '/bin/cat' AS (tkey, tvalue) FROM src a join src b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/heyongqiang/hive_2010-10-27_15-03-01_031_3277101842984747988/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_13-17-27_114_2192122567440751595/-mr-10000 0 val_0 0 val_0 0 val_0 Index: src/test/results/clientpositive/bucketmapjoin_negative2.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin_negative2.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin_negative2.q.out (working copy) @@ -79,7 +79,7 @@ TableScan alias: b GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -129,9 +129,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -141,12 +141,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213147 + transient_lastDdlTime 1288895258 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -156,9 +156,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -170,12 +170,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213145 + transient_lastDdlTime 1288895253 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -187,12 +187,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213145 + transient_lastDdlTime 1288895253 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -204,14 +204,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002 - destination: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -221,28 +221,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213147 + transient_lastDdlTime 1288895258 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -253,12 +253,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213147 + transient_lastDdlTime 1288895258 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -266,9 +266,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002 [pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002] Path -> Partition: - pfile:/data/users/heyongqiang/hive-indexing/build/ql/scratchdir/hive_2010-10-27_13-59-07_847_8947357711223644876/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-27-38_256_1586362839830193186/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -279,12 +279,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213147 + transient_lastDdlTime 1288895258 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -295,12 +295,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/heyongqiang/hive-indexing/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288213147 + transient_lastDdlTime 1288895258 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result Index: src/test/results/clientpositive/join28.q.out =================================================================== --- src/test/results/clientpositive/join28.q.out (revision 1032800) +++ src/test/results/clientpositive/join28.q.out (working copy) @@ -45,7 +45,7 @@ subq:x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} 1 @@ -65,7 +65,7 @@ predicate: expr: (hr = 11) type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col0} 1 {value} @@ -147,7 +147,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-58_634_5247474048070349832/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-55_834_2268567445499795563/-ext-10000 Stage: Stage-0 Move Operator @@ -165,7 +165,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-37-58_634_5247474048070349832/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_18-59-55_834_2268567445499795563/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -203,11 +203,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-38-05_164_4212085229910053721/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-00-06_183_3300125804401305378/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-38-05_164_4212085229910053721/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-00-06_183_3300125804401305378/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] 128 val_128 Index: src/test/results/clientpositive/join37.q.out =================================================================== --- src/test/results/clientpositive/join37.q.out (revision 1032800) +++ src/test/results/clientpositive/join37.q.out (working copy) @@ -36,7 +36,7 @@ x TableScan alias: x - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -109,7 +109,7 @@ Move Operator files: hdfs directory: true - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-56-29_885_2306134457319021637/-ext-10000 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-13-32_000_1610023662261116132/-ext-10000 Stage: Stage-0 Move Operator @@ -127,7 +127,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-56-29_885_2306134457319021637/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_12-13-32_000_1610023662261116132/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 @@ -158,11 +158,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-34_279_5727082010829048557/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-42_031_8017880576006388638/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-56-34_279_5727082010829048557/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_12-13-42_031_8017880576006388638/-mr-10000 POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/bucketmapjoin4.q.out =================================================================== --- src/test/results/clientpositive/bucketmapjoin4.q.out (revision 1032800) +++ src/test/results/clientpositive/bucketmapjoin4.q.out (working copy) @@ -104,7 +104,7 @@ TableScan alias: b GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -117,10 +117,10 @@ Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - b {pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + b {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 Stage: Stage-1 Map Reduce @@ -162,9 +162,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -174,12 +174,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205575 + transient_lastDdlTime 1288894960 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -189,9 +189,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -203,12 +203,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205572 + transient_lastDdlTime 1288894953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -220,12 +220,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205572 + transient_lastDdlTime 1288894953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -237,14 +237,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002 - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -254,28 +254,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205575 + transient_lastDdlTime 1288894960 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -286,12 +286,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205575 + transient_lastDdlTime 1288894960 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -299,9 +299,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002 [pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-52-55_191_6284354312574445049/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-22-40_615_6683611377161445328/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -312,12 +312,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205575 + transient_lastDdlTime 1288894960 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -328,12 +328,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205575 + transient_lastDdlTime 1288894960 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -359,11 +359,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-02_738_4577457943531711045/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-22-55_207_7676433402628717113/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-02_738_4577457943531711045/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-22-55_207_7676433402628717113/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin)b.FieldSchema(name:value, type:string, comment:null), ] @@ -410,11 +410,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-16_429_5837890045100564683/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-19_849_7262162116822440/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-16_429_5837890045100564683/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-19_849_7262162116822440/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -453,14 +453,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-24_119_5713666769406989600/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-30_833_1363058298477291982/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-24_119_5713666769406989600/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-30_833_1363058298477291982/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -522,7 +522,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -535,10 +535,10 @@ Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 Stage: Stage-1 Map Reduce @@ -580,9 +580,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -592,7 +592,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -601,7 +601,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288205596 + transient_lastDdlTime 1288894999 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -611,9 +611,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [b] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -625,12 +625,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205572 + transient_lastDdlTime 1288894953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -642,12 +642,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288205572 + transient_lastDdlTime 1288894953 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -659,14 +659,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002 - destination: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -676,7 +676,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -685,23 +685,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288205596 + transient_lastDdlTime 1288894999 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -712,7 +712,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -721,7 +721,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288205596 + transient_lastDdlTime 1288894999 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -729,9 +729,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002 [pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002] Path -> Partition: - pfile:/data/users/njain/hive-commit1/build/ql/scratchdir/hive_2010-10-27_11-53-27_274_740197042204524306/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-04_11-23-36_325_7813368709164240016/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -742,7 +742,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -751,7 +751,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288205596 + transient_lastDdlTime 1288894999 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -762,7 +762,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/njain/hive-commit1/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -771,7 +771,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288205596 + transient_lastDdlTime 1288894999 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -809,11 +809,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-34_796_7433622090937468625/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-51_993_4668218591132162022/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-34_796_7433622090937468625/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-23-51_993_4668218591132162022/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -896,11 +896,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-48_705_2477192820764361821/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-24-23_108_4512355766611583689/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-48_705_2477192820764361821/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-24-23_108_4512355766611583689/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] @@ -963,14 +963,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-55_192_5237681680689076654/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-24-35_316_1675503630341751532/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/njain/hive_2010-10-27_11-53-55_192_5237681680689076654/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-04_11-24-35_316_1675503630341751532/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] Index: src/test/results/clientpositive/union22.q.out =================================================================== --- src/test/results/clientpositive/union22.q.out (revision 1032800) +++ src/test/results/clientpositive/union22.q.out (working copy) @@ -119,7 +119,7 @@ expr: k4 type: string outputColumnNames: _col1, _col3, _col4 - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {k1} {k2} 1 {_col3} {_col4} @@ -162,7 +162,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/sdong/hive_2010-10-29_17-14-01_563_4403090701419949795/-mr-10002 + directory: file:/tmp/liyintang/hive_2010-11-08_21-11-17_290_439394066953047897/-mr-10002 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -178,9 +178,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22/ds=1 [null-subquery2:subq-subquery2:a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22/ds=1 [null-subquery2:subq-subquery2:a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22/ds=1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22/ds=1 Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat @@ -193,7 +193,7 @@ columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22 name dst_union22 numFiles 1 numPartitions 1 @@ -203,7 +203,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1288397636 + transient_lastDdlTime 1289279470 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -214,7 +214,7 @@ columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22 name dst_union22 numFiles 1 numPartitions 1 @@ -224,7 +224,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1288397636 + transient_lastDdlTime 1289279470 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dst_union22 name: dst_union22 @@ -232,7 +232,7 @@ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2010-10-29_17-14-01_563_4403090701419949795/-mr-10002 + file:/tmp/liyintang/hive_2010-11-08_21-11-17_290_439394066953047897/-mr-10002 Select Operator expressions: expr: _col0 @@ -275,10 +275,10 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000 NumFilesPerFileSink: 1 Static Partition Specification: ds=2/ - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -288,7 +288,7 @@ columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22 name dst_union22 numFiles 1 numPartitions 1 @@ -298,7 +298,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1288397636 + transient_lastDdlTime 1289279470 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dst_union22 TotalFiles: 1 @@ -344,10 +344,10 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000 NumFilesPerFileSink: 1 Static Partition Specification: ds=2/ - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -357,7 +357,7 @@ columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22 name dst_union22 numFiles 1 numPartitions 1 @@ -367,7 +367,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1288397636 + transient_lastDdlTime 1289279470 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dst_union22 TotalFiles: 1 @@ -375,10 +375,10 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - file:/tmp/sdong/hive_2010-10-29_17-14-01_563_4403090701419949795/-mr-10002 [file:/tmp/sdong/hive_2010-10-29_17-14-01_563_4403090701419949795/-mr-10002] - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22_delta/ds=1 [null-subquery1:subq-subquery1:dst_union22_delta] + file:/tmp/liyintang/hive_2010-11-08_21-11-17_290_439394066953047897/-mr-10002 [file:/tmp/liyintang/hive_2010-11-08_21-11-17_290_439394066953047897/-mr-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22_delta/ds=1 [null-subquery1:subq-subquery1:dst_union22_delta] Path -> Partition: - file:/tmp/sdong/hive_2010-10-29_17-14-01_563_4403090701419949795/-mr-10002 + file:/tmp/liyintang/hive_2010-11-08_21-11-17_290_439394066953047897/-mr-10002 Partition base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -394,7 +394,7 @@ columns _col0,_col1,_col10,_col11 columns.types string,string,string,string escape.delim \ - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22_delta/ds=1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22_delta/ds=1 Partition base file name: ds=1 input format: org.apache.hadoop.mapred.TextInputFormat @@ -407,7 +407,7 @@ columns.types string:string:string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22_delta + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22_delta name dst_union22_delta numFiles 1 numPartitions 1 @@ -417,7 +417,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17436 - transient_lastDdlTime 1288397641 + transient_lastDdlTime 1289279477 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -428,7 +428,7 @@ columns.types string:string:string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22_delta + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22_delta name dst_union22_delta numFiles 1 numPartitions 1 @@ -438,7 +438,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 17436 - transient_lastDdlTime 1288397641 + transient_lastDdlTime 1289279477 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dst_union22_delta name: dst_union22_delta @@ -449,7 +449,7 @@ partition: ds 2 replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -459,7 +459,7 @@ columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dst_union22 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dst_union22 name dst_union22 numFiles 1 numPartitions 1 @@ -469,14 +469,14 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1288397636 + transient_lastDdlTime 1289279470 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dst_union22 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10001 Stage: Stage-4 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_17-14-01_563_4403090701419949795/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_21-11-17_290_439394066953047897/-ext-10000/ PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') select * from @@ -525,11 +525,11 @@ PREHOOK: query: select * from dst_union22 where ds = '2' order by k1 PREHOOK: type: QUERY PREHOOK: Input: default@dst_union22@ds=2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_17-14-11_051_2672122077731872074/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_21-11-31_910_3533267126260963174/-mr-10000 POSTHOOK: query: select * from dst_union22 where ds = '2' order by k1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dst_union22@ds=2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_17-14-11_051_2672122077731872074/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_21-11-31_910_3533267126260963174/-mr-10000 POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dst_union22 PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] Index: src/test/results/clientpositive/join32.q.out =================================================================== --- src/test/results/clientpositive/join32.q.out (revision 1032800) +++ src/test/results/clientpositive/join32.q.out (working copy) @@ -41,7 +41,7 @@ TableScan alias: x GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} @@ -73,7 +73,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/sdong/hive_2010-10-29_15-38-59_184_2779990290442472644/-mr-10003 + directory: file:/tmp/liyintang/hive_2010-11-08_19-01-24_245_1589831135817422269/-mr-10003 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -89,9 +89,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src [y] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src [y] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -102,12 +102,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389296 + transient_lastDdlTime 1289268011 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -118,12 +118,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/src + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/src name src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288389296 + transient_lastDdlTime 1289268011 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src name: src @@ -149,7 +149,7 @@ predicate: expr: (hr = 11) type: boolean - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {_col5} {_col0} 1 {value} @@ -162,7 +162,7 @@ Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2010-10-29_15-38-59_184_2779990290442472644/-mr-10003 + file:/tmp/liyintang/hive_2010-11-08_19-01-24_245_1589831135817422269/-mr-10003 Select Operator expressions: expr: _col0 @@ -205,9 +205,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -217,12 +217,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391939 + transient_lastDdlTime 1289271684 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -232,9 +232,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - file:/tmp/sdong/hive_2010-10-29_15-38-59_184_2779990290442472644/-mr-10003 [file:/tmp/sdong/hive_2010-10-29_15-38-59_184_2779990290442472644/-mr-10003] + file:/tmp/liyintang/hive_2010-11-08_19-01-24_245_1589831135817422269/-mr-10003 [file:/tmp/liyintang/hive_2010-11-08_19-01-24_245_1589831135817422269/-mr-10003] Path -> Partition: - file:/tmp/sdong/hive_2010-10-29_15-38-59_184_2779990290442472644/-mr-10003 + file:/tmp/liyintang/hive_2010-11-08_19-01-24_245_1589831135817422269/-mr-10003 Partition base file name: -mr-10003 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -258,14 +258,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -275,28 +275,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391939 + transient_lastDdlTime 1289271684 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -307,12 +307,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391939 + transient_lastDdlTime 1289271684 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 TotalFiles: 1 @@ -320,9 +320,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_15-38-59_184_2779990290442472644/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_19-01-24_245_1589831135817422269/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -333,12 +333,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391939 + transient_lastDdlTime 1289271684 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -349,12 +349,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/dest_j1 + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/dest_j1 name dest_j1 serialization.ddl struct dest_j1 { string key, string value, string val2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288391939 + transient_lastDdlTime 1289271684 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dest_j1 name: dest_j1 @@ -384,11 +384,11 @@ PREHOOK: query: select * from dest_j1 x order by x.key PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-39-08_387_2091136014387803830/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-01-38_997_5755790132038550537/-mr-10000 POSTHOOK: query: select * from dest_j1 x order by x.key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest_j1 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_15-39-08_387_2091136014387803830/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_19-01-38_997_5755790132038550537/-mr-10000 POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest_j1.value SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] Index: src/test/results/clientpositive/stats11.q.out =================================================================== --- src/test/results/clientpositive/stats11.q.out (revision 1032800) +++ src/test/results/clientpositive/stats11.q.out (working copy) @@ -104,7 +104,7 @@ TableScan alias: b GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} {ds} @@ -117,10 +117,10 @@ Alias Bucket Base File Name Mapping: b {srcbucket20.txt=[srcbucket20.txt, srcbucket22.txt], srcbucket21.txt=[srcbucket21.txt, srcbucket23.txt]} Alias Bucket File Name Mapping: - b {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} + b {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt, pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt 1 Stage: Stage-1 Map Reduce @@ -169,9 +169,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -181,12 +181,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395826 + transient_lastDdlTime 1289277098 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -196,9 +196,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin [a] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin [a] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin Partition base file name: srcbucket_mapjoin input format: org.apache.hadoop.mapred.TextInputFormat @@ -210,12 +210,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395819 + transient_lastDdlTime 1289277090 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -227,12 +227,12 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin name srcbucket_mapjoin serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395819 + transient_lastDdlTime 1289277090 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin name: srcbucket_mapjoin @@ -244,14 +244,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -261,28 +261,28 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395826 + transient_lastDdlTime 1289277098 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -293,12 +293,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395826 + transient_lastDdlTime 1289277098 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -306,9 +306,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-43-46_394_6815278039409347690/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-31-38_084_5882852540626894180/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -319,12 +319,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395826 + transient_lastDdlTime 1289277098 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -335,12 +335,12 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result serialization.ddl struct bucketmapjoin_tmp_result { string key, string value1, string value2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395826 + transient_lastDdlTime 1289277098 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -368,11 +368,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-43-57_636_6913530990309837371/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-31-53_537_6021440350348908908/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-43-57_636_6913530990309837371/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-31-53_537_6021440350348908908/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_tmp_result.key SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin)a.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part)b.FieldSchema(name:value, type:string, comment:null), ] @@ -421,11 +421,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-15_220_6414939925632720078/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-19_217_370638698081651509/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-15_220_6414939925632720078/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-19_217_370638698081651509/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -464,14 +464,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-22_991_5574834826319442482/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-30_131_6677259791124778332/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-22_991_5574834826319442482/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-30_131_6677259791124778332/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -533,7 +533,7 @@ TableScan alias: a GatherStats: false - JDBM Sink Operator + HashTable Sink Operator condition expressions: 0 {key} {value} 1 {value} {ds} @@ -546,12 +546,12 @@ Alias Bucket Base File Name Mapping: a {srcbucket20.txt=[srcbucket20.txt], srcbucket21.txt=[srcbucket21.txt], srcbucket22.txt=[srcbucket20.txt], srcbucket23.txt=[srcbucket21.txt]} Alias Bucket File Name Mapping: - a {pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} + a {pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket20.txt], pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt=[pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin/srcbucket21.txt]} Alias Bucket Output File Name Mapping: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket20.txt 0 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket21.txt 1 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket22.txt 2 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08/srcbucket23.txt 3 Stage: Stage-1 Map Reduce @@ -600,9 +600,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -612,7 +612,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -621,7 +621,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288395855 + transient_lastDdlTime 1289277139 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -631,9 +631,9 @@ Map Reduce Local Work Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 [b] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 + pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -647,13 +647,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395819 + transient_lastDdlTime 1289277090 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -665,13 +665,13 @@ columns.types int:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/srcbucket_mapjoin_part + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/srcbucket_mapjoin_part name srcbucket_mapjoin_part partition_columns ds serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1288395819 + transient_lastDdlTime 1289277090 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcbucket_mapjoin_part name: srcbucket_mapjoin_part @@ -683,14 +683,14 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002 - destination: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002 + destination: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true - source: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10000 + source: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -700,7 +700,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -709,23 +709,23 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288395855 + transient_lastDdlTime 1289277139 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result - tmp directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10001 + tmp directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10000 + directory: pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -736,7 +736,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -745,7 +745,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288395855 + transient_lastDdlTime 1289277139 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result TotalFiles: 1 @@ -753,9 +753,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002 [pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002] + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002 [pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002] Path -> Partition: - pfile:/data/users/sdong/www/hive-trunk/build/ql/scratchdir/hive_2010-10-29_16-44-26_534_2407093348880744592/-ext-10002 + pfile:/data/users/liyintang/hive-1754/build/ql/scratchdir/hive_2010-11-08_20-32-35_603_7737558187075598476/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -766,7 +766,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -775,7 +775,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288395855 + transient_lastDdlTime 1289277139 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -786,7 +786,7 @@ columns.types string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/hive-trunk/build/ql/test/data/warehouse/bucketmapjoin_tmp_result + location pfile:/data/users/liyintang/hive-1754/build/ql/test/data/warehouse/bucketmapjoin_tmp_result name bucketmapjoin_tmp_result numFiles 1 numPartitions 0 @@ -795,7 +795,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 8983 - transient_lastDdlTime 1288395855 + transient_lastDdlTime 1289277139 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: bucketmapjoin_tmp_result name: bucketmapjoin_tmp_result @@ -835,11 +835,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-39_626_4157954330716298219/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-53_587_6517504932310697157/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-39_626_4157954330716298219/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-32-53_587_6517504932310697157/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ] @@ -924,11 +924,11 @@ PREHOOK: query: select count(1) from bucketmapjoin_tmp_result PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_tmp_result -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-57_762_5091416448208285883/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-33-19_305_2833852308539498006/-mr-10000 POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_tmp_result -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-44-57_762_5091416448208285883/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-33-19_305_2833852308539498006/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] @@ -991,14 +991,14 @@ PREHOOK: type: QUERY PREHOOK: Input: default@bucketmapjoin_hash_result_1 PREHOOK: Input: default@bucketmapjoin_hash_result_2 -PREHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-45-05_839_2112240909542356817/-mr-10000 +PREHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-33-31_287_7076595751326509650/-mr-10000 POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2 from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b on a.key = b.key POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketmapjoin_hash_result_1 POSTHOOK: Input: default@bucketmapjoin_hash_result_2 -POSTHOOK: Output: file:/tmp/sdong/hive_2010-10-29_16-45-05_839_2112240909542356817/-mr-10000 +POSTHOOK: Output: file:/tmp/liyintang/hive_2010-11-08_20-33-31_287_7076595751326509650/-mr-10000 POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ] Index: src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (working copy) @@ -26,13 +26,13 @@ import java.util.Map; import java.util.Stack; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.MapredLocalTask; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -97,9 +97,10 @@ if(localwork != null){ //get the context info and set up the shared tmp URI Context ctx = physicalContext.getContext(); - String tmpFileURI = ctx.getLocalTmpFileURI()+Path.SEPARATOR+"JDBM-"+currTask.getId(); + String tmpFileURI = Utilities.generateTmpURI(ctx.getLocalTmpFileURI(), currTask.getId()); localwork.setTmpFileURI(tmpFileURI); - mapredWork.setTmpHDFSFileURI(ctx.getMRTmpFileURI()+Path.SEPARATOR+"JDBM-"+currTask.getId()); + String hdfsTmpURI = Utilities.generateTmpURI(ctx.getMRTmpFileURI(), currTask.getId()); + mapredWork.setTmpHDFSFileURI(hdfsTmpURI); //create a task for this local work; right now, this local work is shared //by the original MapredTask and this new generated MapredLocalTask. MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork, Index: src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java (working copy) @@ -23,8 +23,8 @@ import java.util.List; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.JDBMDummyOperator; -import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator; +import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; @@ -35,8 +35,8 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.optimizer.physical.MapJoinResolver.LocalMapJoinProcCtx; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; -import org.apache.hadoop.hive.ql.plan.JDBMSinkDesc; +import org.apache.hadoop.hive.ql.plan.HashTableDummyDesc; +import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.TableDesc; @@ -78,11 +78,11 @@ } MapJoinOperator mapJoinOp = (MapJoinOperator) nd; - //create an new operator: JDBMSinkOperator - JDBMSinkDesc jdbmSinkDesc = new JDBMSinkDesc(mapJoinOp.getConf()); - JDBMSinkOperator jdbmSinkOp =(JDBMSinkOperator)OperatorFactory.get(jdbmSinkDesc); + HashTableSinkDesc hashTableSinkDesc = new HashTableSinkDesc(mapJoinOp.getConf()); + HashTableSinkOperator hashTableSinkOp =(HashTableSinkOperator)OperatorFactory.get(hashTableSinkDesc); + //get the last operator for processing big tables int bigTable = mapJoinOp.getConf().getPosBigTable(); Byte[] order = mapJoinOp.getConf().getTagOrder(); @@ -90,7 +90,7 @@ Operator bigOp = mapJoinOp.getParentOperators().get(bigTable); - //the parent ops for jdbmSinkOp + //the parent ops for hashTableSinkOp List> smallTablesParentOp= new ArrayList>(); List> dummyOperators= new ArrayList>(); @@ -103,14 +103,14 @@ } Operator parent = parentsOp.get(i); - //let jdbmOp be the child of this parent - parent.replaceChild(mapJoinOp, jdbmSinkOp); + //let hashtable Op be the child of this parent + parent.replaceChild(mapJoinOp, hashTableSinkOp); //keep the parent id correct smallTablesParentOp.add(parent); - //create an new operator: JDBMDummyOpeator, which share the table desc - JDBMDummyDesc desc = new JDBMDummyDesc(); - JDBMDummyOperator dummyOp =(JDBMDummyOperator)OperatorFactory.get(desc); + //create an new operator: HashTable DummyOpeator, which share the table desc + HashTableDummyDesc desc = new HashTableDummyDesc(); + HashTableDummyOperator dummyOp =(HashTableDummyOperator)OperatorFactory.get(desc); TableDesc tbl; if(parent.getSchema()==null){ @@ -140,7 +140,7 @@ } - jdbmSinkOp.setParentOperators(smallTablesParentOp); + hashTableSinkOp.setParentOperators(smallTablesParentOp); for(Operator op: dummyOperators){ context.addDummyParentOp(op); } Index: src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/JDBMDummyOperator.java (working copy) @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec; - -import java.io.Serializable; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.serde2.Deserializer; - -public class JDBMDummyOperator extends Operator implements Serializable{ - - @Override - protected void initializeOp(Configuration hconf) throws HiveException { - TableDesc tbl = this.getConf().getTbl(); - try{ - Deserializer serde = tbl.getDeserializerClass().newInstance(); - serde.initialize(hconf, tbl.getProperties()); - - - this.outputObjInspector = serde.getObjectInspector(); - initializeChildren(hconf); - }catch(Exception e){ - LOG.error("Generating output obj inspector from dummy object error"); - e.printStackTrace(); - } - - } - - @Override - public void processOp(Object row, int tag) throws HiveException{ - throw new HiveException(); - } - - @Override - public void closeOp(boolean abort) throws HiveException{ - } - @Override - public String getName() { - return "JDBMDUMMY"; - } - - @Override - public int getType() { - return OperatorType.JDBMDUMMY; - } - -} Index: src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (working copy) @@ -29,7 +29,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; Index: src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (working copy) @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.ql.plan.MapredLocalWork.BucketMapJoinContext; import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; Index: src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/HashTableDummyOperator.java (revision 0) @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec; + +import java.io.Serializable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.HashTableDummyDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.serde2.Deserializer; + +public class HashTableDummyOperator extends Operator implements Serializable { + private static final long serialVersionUID = 1L; + + @Override + protected void initializeOp(Configuration hconf) throws HiveException { + TableDesc tbl = this.getConf().getTbl(); + try { + Deserializer serde = tbl.getDeserializerClass().newInstance(); + serde.initialize(hconf, tbl.getProperties()); + this.outputObjInspector = serde.getObjectInspector(); + initializeChildren(hconf); + } catch (Exception e) { + LOG.error("Generating output obj inspector from dummy object error"); + e.printStackTrace(); + } + } + + @Override + public void processOp(Object row, int tag) throws HiveException { + throw new HiveException(); + } + + @Override + public void closeOp(boolean abort) throws HiveException { + } + + @Override + public String getName() { + return "HASHTABLEDUMMY"; + } + + @Override + public int getType() { + return OperatorType.HASHTABLEDUMMY; + } + +} Index: src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java (working copy) @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -164,7 +163,7 @@ // reset rowcontainer's serde, objectinspector, and tableDesc. for (int i = 0; i < numAliases; i++) { Byte alias = conf.getTagOrder()[i]; - RowContainer> rc = joinOp.storage.get(Byte + RowContainer> rc = (RowContainer)joinOp.storage.get(Byte .valueOf((byte) i)); if (rc != null) { rc.setSerDe(tblSerializers.get((byte) i), skewKeysTableObjectInspector @@ -178,7 +177,7 @@ if (skewKeyInCurrentGroup) { String specPath = conf.getBigKeysDirMap().get((byte) currBigKeyTag); - RowContainer> bigKey = joinOp.storage.get(Byte + RowContainer> bigKey = (RowContainer)joinOp.storage.get(Byte .valueOf((byte) currBigKeyTag)); Path outputPath = getOperatorOutputPath(specPath); FileSystem destFs = outputPath.getFileSystem(hconf); @@ -188,7 +187,7 @@ if (((byte) i) == currBigKeyTag) { continue; } - RowContainer> values = joinOp.storage.get(Byte + RowContainer> values = (RowContainer)joinOp.storage.get(Byte .valueOf((byte) i)); if (values != null) { specPath = conf.getSmallKeysDirMap().get((byte) currBigKeyTag).get( @@ -216,7 +215,7 @@ skewKeyInCurrentGroup = false; for (int i = 0; i < numAliases; i++) { - RowContainer> rc = joinOp.storage.get(Byte + RowContainer> rc = (RowContainer)joinOp.storage.get(Byte .valueOf((byte) i)); if (rc != null) { rc.setKeyObject(dummyKey); Index: src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (revision 0) @@ -0,0 +1,416 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec; + +import java.io.File; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.persistence.AbstractMapJoinKey; +import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectValue; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer; +import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.util.ReflectionUtils; + + +public class HashTableSinkOperator extends TerminalOperator implements + Serializable { + private static final long serialVersionUID = 1L; + private static final Log LOG = LogFactory.getLog(HashTableSinkOperator.class.getName()); + + // from abstract map join operator + /** + * The expressions for join inputs's join keys. + */ + protected transient Map> joinKeys; + /** + * The ObjectInspectors for the join inputs's join keys. + */ + protected transient Map> joinKeysObjectInspectors; + /** + * The standard ObjectInspectors for the join inputs's join keys. + */ + protected transient Map> joinKeysStandardObjectInspectors; + + protected transient int posBigTableTag = -1; // one of the tables that is not in memory + protected transient int posBigTableAlias = -1; // one of the tables that is not in memory + transient int mapJoinRowsKey; // rows for a given key + + protected transient RowContainer> emptyList = null; + + transient int numMapRowsRead; + protected transient int totalSz; // total size of the composite object + transient boolean firstRow; + /** + * The filters for join + */ + protected transient Map> joinFilters; + + protected transient int numAliases; // number of aliases + /** + * The expressions for join outputs. + */ + protected transient Map> joinValues; + /** + * The ObjectInspectors for the join inputs. + */ + protected transient Map> joinValuesObjectInspectors; + /** + * The ObjectInspectors for join filters. + */ + protected transient Map> joinFilterObjectInspectors; + /** + * The standard ObjectInspectors for the join inputs. + */ + protected transient Map> joinValuesStandardObjectInspectors; + + protected transient Map> rowContainerStandardObjectInspectors; + + protected transient Byte[] order; // order in which the results should + Configuration hconf; + protected transient Byte alias; + protected transient Map spillTableDesc; // spill tables are + + protected transient Map> mapJoinTables; + protected transient boolean noOuterJoin; + + private long rowNumber = 0; + protected transient LogHelper console; + + public static class HashTableSinkObjectCtx { + ObjectInspector standardOI; + SerDe serde; + TableDesc tblDesc; + Configuration conf; + + /** + * @param standardOI + * @param serde + */ + public HashTableSinkObjectCtx(ObjectInspector standardOI, SerDe serde, TableDesc tblDesc, + Configuration conf) { + this.standardOI = standardOI; + this.serde = serde; + this.tblDesc = tblDesc; + this.conf = conf; + } + + /** + * @return the standardOI + */ + public ObjectInspector getStandardOI() { + return standardOI; + } + + /** + * @return the serde + */ + public SerDe getSerDe() { + return serde; + } + + public TableDesc getTblDesc() { + return tblDesc; + } + + public Configuration getConf() { + return conf; + } + + } + + private static final transient String[] FATAL_ERR_MSG = { + null, // counter value 0 means no error + "Mapside join size exceeds hive.mapjoin.maxsize. " + + "Please increase that or remove the mapjoin hint."}; + private final int metadataKeyTag = -1; + transient int[] metadataValueTag; + transient int maxMapJoinSize; + + + public HashTableSinkOperator() { + // super(); + console = new LogHelper(LOG, true); + } + + public HashTableSinkOperator(MapJoinOperator mjop) { + this.conf = new HashTableSinkDesc(mjop.getConf()); + console = new LogHelper(LOG); + } + + + @Override + protected void initializeOp(Configuration hconf) throws HiveException { + + maxMapJoinSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAXMAPJOINSIZE); + + numMapRowsRead = 0; + firstRow = true; + + // for small tables only; so get the big table position first + posBigTableTag = conf.getPosBigTable(); + + order = conf.getTagOrder(); + + posBigTableAlias = order[posBigTableTag]; + + // initialize some variables, which used to be initialized in CommonJoinOperator + numAliases = conf.getExprs().size(); + this.hconf = hconf; + totalSz = 0; + + noOuterJoin = conf.isNoOuterJoin(); + + // process join keys + joinKeys = new HashMap>(); + JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), order, posBigTableAlias); + joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys, + inputObjInspectors, posBigTableAlias); + joinKeysStandardObjectInspectors = JoinUtil.getStandardObjectInspectors( + joinKeysObjectInspectors, posBigTableAlias); + + // process join values + joinValues = new HashMap>(); + JoinUtil.populateJoinKeyValue(joinValues, conf.getExprs(), order, posBigTableAlias); + joinValuesObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinValues, + inputObjInspectors, posBigTableAlias); + joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors( + joinValuesObjectInspectors, posBigTableAlias); + + // process join filters + joinFilters = new HashMap>(); + JoinUtil.populateJoinKeyValue(joinFilters, conf.getFilters(), order, posBigTableAlias); + joinFilterObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinFilters, + inputObjInspectors, posBigTableAlias); + + if (noOuterJoin) { + rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors; + } else { + Map> rowContainerObjectInspectors = new HashMap>(); + for (Byte alias : order) { + if (alias == posBigTableAlias) { + continue; + } + ArrayList rcOIs = new ArrayList(); + rcOIs.addAll(joinValuesObjectInspectors.get(alias)); + // for each alias, add object inspector for boolean as the last element + rcOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + rowContainerObjectInspectors.put(alias, rcOIs); + } + rowContainerStandardObjectInspectors = getStandardObjectInspectors(rowContainerObjectInspectors); + } + + metadataValueTag = new int[numAliases]; + for (int pos = 0; pos < numAliases; pos++) { + metadataValueTag[pos] = -1; + } + + mapJoinTables = new HashMap>(); + + // initialize the hash tables for other tables + for (Byte pos : order) { + if (pos == posBigTableTag) { + continue; + } + + HashMapWrapper hashTable = new HashMapWrapper(); + + + mapJoinTables.put(pos, hashTable); + } + } + + protected static HashMap> getStandardObjectInspectors( + Map> aliasToObjectInspectors) { + HashMap> result = new HashMap>(); + for (Entry> oiEntry : aliasToObjectInspectors.entrySet()) { + Byte alias = oiEntry.getKey(); + List oiList = oiEntry.getValue(); + ArrayList fieldOIList = new ArrayList(oiList.size()); + for (int i = 0; i < oiList.size(); i++) { + fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList.get(i), + ObjectInspectorCopyOption.WRITABLE)); + } + result.put(alias, fieldOIList); + } + return result; + } + + public void generateMapMetaData() throws Exception { + TableDesc keyTableDesc = conf.getKeyTblDesc(); + SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), + null); + keySerializer.initialize(null, keyTableDesc.getProperties()); + + MapJoinMetaData.clear(); + MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx( + ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf)); + } + + /* + * This operator only process small tables Read the key/value pairs Load them into hashtable + */ + @Override + public void processOp(Object row, int tag) throws HiveException { + // let the mapJoinOp process these small tables + try { + if (firstRow) { + // generate the map metadata + generateMapMetaData(); + firstRow = false; + } + alias = order[tag]; + // alias = (byte)tag; + + // compute keys and values as StandardObjects + AbstractMapJoinKey keyMap = JoinUtil.computeMapJoinKeys(row, joinKeys.get(alias), + joinKeysObjectInspectors.get(alias)); + + Object[] value = JoinUtil.computeMapJoinValues(row, joinValues.get(alias), + joinValuesObjectInspectors.get(alias), joinFilters.get(alias), joinFilterObjectInspectors + .get(alias), noOuterJoin); + + HashMapWrapper hashTable = mapJoinTables + .get((byte) tag); + + MapJoinObjectValue o = hashTable.get(keyMap); + MapJoinRowContainer res = null; + + boolean needNewKey = true; + if (o == null) { + res = new MapJoinRowContainer(); + res.add(value); + + if (metadataValueTag[tag] == -1) { + metadataValueTag[tag] = order[tag]; + + TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); + SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc + .getDeserializerClass(), null); + valueSerDe.initialize(null, valueTableDesc.getProperties()); + + MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]), new HashTableSinkObjectCtx( + ObjectInspectorUtils.getStandardObjectInspector(valueSerDe.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf)); + } + + // Construct externalizable objects for key and value + if (needNewKey) { + MapJoinObjectValue valueObj = new MapJoinObjectValue(metadataValueTag[tag], res); + rowNumber++; + hashTable.put(keyMap, valueObj); + } + + } else { + res = o.getObj(); + res.add(value); + } + + } catch (Exception e) { + e.printStackTrace(); + throw new HiveException(e); + } + + } + + + @Override + public void closeOp(boolean abort) throws HiveException { + try { + if (mapJoinTables != null) { + // get tmp file URI + String tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); + LOG.info("Get TMP URI: " + tmpURI); + long fileLength; + for (Map.Entry> hashTables : mapJoinTables + .entrySet()) { + // get the key and value + Byte tag = hashTables.getKey(); + HashMapWrapper hashTable = hashTables.getValue(); + + // get current input file name + String bigBucketFileName = this.getExecContext().getCurrentBigBucketFile(); + if (bigBucketFileName == null || bigBucketFileName.length() == 0) { + bigBucketFileName = "-"; + } + // get the tmp URI path; it will be a hdfs path if not local mode + String tmpURIPath = Utilities.generatePath(tmpURI, tag, bigBucketFileName); + console.printInfo(Utilities.now() + "\tDump the hashtable into file: " + tmpURIPath); + // get the hashtable file and path + Path path = new Path(tmpURIPath); + FileSystem fs = path.getFileSystem(hconf); + File file = new File(path.toUri().getPath()); + fs.create(path); + + fileLength = hashTable.flushMemoryCacheToPersistent(file); + + console.printInfo(Utilities.now() + "\t Processing rows: " + rowNumber + "\t key number:" + + hashTable.size()); + console.printInfo("Upload 1 File to: " + tmpURIPath + " File size: " + fileLength); + + hashTable.close(); + } + } + + super.closeOp(abort); + } catch (Exception e) { + LOG.error("Generate Hashtable error"); + e.printStackTrace(); + } + } + + /** + * Implements the getName function for the Node Interface. + * + * @return the name of the operator + */ + @Override + public String getName() { + return "HASHTABLESINK"; + } + + @Override + public int getType() { + return OperatorType.HASHTABLESINK; + } + + + +} Index: src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (working copy) @@ -26,11 +26,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.persistence.AbstractMapJoinKey; import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -69,7 +69,6 @@ transient boolean firstRow; - private static final int NOTSKIPBIGTABLE = -1; public AbstractMapJoinOperator() { } @@ -149,4 +148,21 @@ return false; } + // returns true if there are elements in key list and any of them is null + protected boolean hasAnyNulls(Object[] key) { + if (key != null && key.length> 0) { + for (Object k : key) { + if (k == null) { + return true; + } + } + } + return false; + } + + // returns true if there are elements in key list and any of them is null + protected boolean hasAnyNulls(AbstractMapJoinKey key) { + return key.hasAnyNulls(); + } + } Index: src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (working copy) @@ -38,6 +38,7 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.StringUtils; + /** * ExecMapper. * @@ -58,7 +59,6 @@ private long numRows = 0; private long nextCntr = 1; private MapredLocalWork localWork = null; - private boolean isLogInfoEnabled = false; private final ExecMapperContext execContext = new ExecMapperContext(); @@ -68,14 +68,12 @@ memoryMXBean = ManagementFactory.getMemoryMXBean(); l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); - isLogInfoEnabled = l4j.isInfoEnabled(); - try { l4j.info("conf classpath = " + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs())); l4j.info("thread classpath = " - + Arrays.asList(((URLClassLoader) Thread.currentThread() - .getContextClassLoader()).getURLs())); + + Arrays.asList(((URLClassLoader) Thread.currentThread().getContextClassLoader()) + .getURLs())); } catch (Exception e) { l4j.info("cannot get classpath: " + e.getMessage()); } @@ -84,12 +82,13 @@ execContext.setJc(jc); // create map and fetch operators MapredWork mrwork = Utilities.getMapRedWork(job); + mo = new MapOperator(); mo.setConf(mrwork); // initialize map operator mo.setChildren(job); l4j.info(mo.dump(0)); - // initialize map local work + // initialize map local work localWork = mrwork.getMapLocalWork(); execContext.setLocalWork(localWork); @@ -101,16 +100,15 @@ return; } - //The following code is for mapjoin - //initialize all the dummy ops + // The following code is for mapjoin + // initialize all the dummy ops l4j.info("Initializing dummy operator"); List> dummyOps = localWork.getDummyParentOp(); - for(Operator dummyOp : dummyOps){ + for (Operator dummyOp : dummyOps) { dummyOp.setExecContext(execContext); - dummyOp.initialize(jc,null); + dummyOp.initialize(jc, null); } - } catch (Throwable e) { abort = true; if (e instanceof OutOfMemoryError) { @@ -121,10 +119,11 @@ throw new RuntimeException("Map operator initialization failed", e); } } + } - public void map(Object key, Object value, OutputCollector output, - Reporter reporter) throws IOException { + public void map(Object key, Object value, OutputCollector output, Reporter reporter) + throws IOException { if (oc == null) { oc = output; rp = reporter; @@ -140,13 +139,12 @@ } else { // Since there is no concept of a group, we don't invoke // startGroup/endGroup for a mapper - mo.process((Writable)value); - if (isLogInfoEnabled) { + mo.process((Writable) value); + if (l4j.isInfoEnabled()) { numRows++; if (numRows == nextCntr) { long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("ExecMapper: processing " + numRows - + " rows: used memory = " + used_memory); + l4j.info("ExecMapper: processing " + numRows + " rows: used memory = " + used_memory); nextCntr = getNextCntr(numRows); } } @@ -192,11 +190,11 @@ try { mo.close(abort); - //for close the local work - if(localWork != null){ + // for close the local work + if (localWork != null) { List> dummyOps = localWork.getDummyParentOp(); - for(Operator dummyOp : dummyOps){ + for (Operator dummyOp : dummyOps) { dummyOp.close(abort); } } @@ -204,16 +202,15 @@ if (fetchOperators != null) { MapredLocalWork localWork = mo.getConf().getMapLocalWork(); for (Map.Entry entry : fetchOperators.entrySet()) { - Operator forwardOp = localWork - .getAliasToWork().get(entry.getKey()); + Operator forwardOp = localWork.getAliasToWork().get( + entry.getKey()); forwardOp.close(abort); } } - if (isLogInfoEnabled) { + if (l4j.isInfoEnabled()) { long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed(); - l4j.info("ExecMapper: processed " + numRows + " rows: used memory = " - + used_memory); + l4j.info("ExecMapper: processed " + numRows + " rows: used memory = " + used_memory); } reportStats rps = new reportStats(rp); @@ -244,6 +241,8 @@ ExecMapper.done = done; } + + /** * reportStats. * Index: src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (working copy) @@ -410,4 +410,7 @@ return null; } + + + } Index: src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (working copy) @@ -185,7 +185,6 @@ HiveConf.ConfVars.HIVESENDHEARTBEAT); countAfterReport = 0; - assert (inputObjInspectors.length == 1); ObjectInspector rowInspector = inputObjInspectors[0]; // init keyFields Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java (working copy) @@ -24,37 +24,33 @@ import java.io.ObjectOutput; import java.util.ArrayList; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; -import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.io.Writable; + /** * Map Join Object used for both key and value. */ public class MapJoinObjectValue implements Externalizable { protected transient int metadataTag; - protected transient RowContainer obj; - protected transient Configuration conf; - protected int bucketSize; // bucket size for RowContainer - protected Log LOG = LogFactory.getLog(this.getClass().getName()); + protected transient MapJoinRowContainer obj; + + public MapJoinObjectValue() { - bucketSize = 100; // default bucket size + } /** * @param metadataTag * @param obj */ - public MapJoinObjectValue(int metadataTag, RowContainer obj) { + public MapJoinObjectValue(int metadataTag, MapJoinRowContainer obj) { this.metadataTag = metadataTag; this.obj = obj; } @@ -63,12 +59,12 @@ public boolean equals(Object o) { if (o instanceof MapJoinObjectValue) { MapJoinObjectValue mObj = (MapJoinObjectValue) o; + if (mObj.getMetadataTag() == metadataTag) { if ((obj == null) && (mObj.getObj() == null)) { return true; } - if ((obj != null) && (mObj.getObj() != null) - && (mObj.getObj().equals(obj))) { + if ((obj != null) && (mObj.getObj() != null) && (mObj.getObj().equals(obj))) { return true; } } @@ -82,20 +78,16 @@ } @Override - public void readExternal(ObjectInput in) throws IOException, - ClassNotFoundException { + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { try { metadataTag = in.readInt(); // get the tableDesc from the map stored in the mapjoin operator - JDBMSinkObjectCtx ctx = MapJoinMetaData.get( - Integer.valueOf(metadataTag)); + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); int sz = in.readInt(); - RowContainer res = new RowContainer(bucketSize, ctx.getConf()); - res.setSerDe(ctx.getSerDe(), ctx.getStandardOI()); - res.setTableDesc(ctx.getTblDesc()); + MapJoinRowContainer res = new MapJoinRowContainer(); if (sz > 0) { int numCols = in.readInt(); if (numCols > 0) { @@ -104,16 +96,18 @@ val.readFields(in); ArrayList memObj = (ArrayList) ObjectInspectorUtils - .copyToStandardObject(ctx.getSerDe().deserialize(val), ctx - .getSerDe().getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE); + .copyToStandardObject(ctx.getSerDe().deserialize(val), ctx.getSerDe() + .getObjectInspector(), ObjectInspectorCopyOption.WRITABLE); - res.add(memObj); + if (memObj == null) { + res.add(new ArrayList(0).toArray()); + } else { + res.add(memObj.toArray()); + } } - } - else{ - for(int i = 0 ; i (0)); + } else { + for (int i = 0; i < sz; i++) { + res.add(new ArrayList(0).toArray()); } } } @@ -130,17 +124,16 @@ out.writeInt(metadataTag); // get the tableDesc from the map stored in the mapjoin operator - JDBMSinkObjectCtx ctx = MapJoinMetaData.get( - Integer.valueOf(metadataTag)); + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); // Different processing for key and value - RowContainer> v = obj; + MapJoinRowContainer v = obj; out.writeInt(v.size()); if (v.size() > 0) { - ArrayList row = v.first(); - out.writeInt(row.size()); + Object[] row = v.first(); + out.writeInt(row.length); - if (row.size() > 0) { + if (row.length > 0) { for (; row != null; row = v.next()) { Writable outVal = ctx.getSerDe().serialize(row, ctx.getStandardOI()); outVal.write(out); @@ -172,7 +165,7 @@ /** * @return the obj */ - public RowContainer getObj() { + public MapJoinRowContainer getObj() { return obj; } @@ -180,13 +173,8 @@ * @param obj * the obj to set */ - public void setObj(RowContainer obj) { + public void setObj(MapJoinRowContainer obj) { this.obj = obj; } - public void setConf(Configuration conf) { - this.conf = conf; - bucketSize = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE); - } - } Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (working copy) @@ -19,456 +19,233 @@ package org.apache.hadoop.hive.ql.exec.persistence; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.text.NumberFormat; import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerFactory; -import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerOptions; -import org.apache.hadoop.hive.ql.util.jdbm.helper.FastIterator; -import org.apache.hadoop.hive.ql.util.jdbm.htree.HTree; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; + /** - * Simple wrapper for persistent Hashmap implementing only the - * put/get/remove/clear interface. The main memory hash table acts as a cache - * and all put/get will operate on it first. If the size of the main memory hash - * table exceeds a certain threshold, new elements will go into the persistent + * Simple wrapper for persistent Hashmap implementing only the put/get/remove/clear interface. The + * main memory hash table acts as a cache and all put/get will operate on it first. If the size of + * the main memory hash table exceeds a certain threshold, new elements will go into the persistent * hash table. */ -public class HashMapWrapper { +public class HashMapWrapper implements Serializable { protected Log LOG = LogFactory.getLog(this.getClass().getName()); // default threshold for using main memory based HashMap - private static final int THRESHOLD = 25000; + private static final int THRESHOLD = 1000000; + private static final float LOADFACTOR = 0.75f; - private int threshold; // threshold to put data into persistent hash table + private double threshold; // threshold to put data into persistent hash table // instead - private HashMap mHash; // main memory HashMap - private HTree pHash; // persistent HashMap - private RecordManager recman; // record manager required by HTree - private File tmpFile; // temp file holding the persistent data from record - // manager. - private MRU MRUList; // MRU cache entry + private HashMap mHash; // main memory HashMap - /** - * Doubly linked list of value items. Note: this is only used along with - * memory hash table. Persistent hash stores the value directory. - */ - class MRUItem extends DCLLItem { - K key; - V value; - MRUItem(K k, V v) { - key = k; - value = v; - } - } + protected transient LogHelper console; + + private File dumpFile; + public static MemoryMXBean memoryMXBean; + private long maxMemory; + private long currentMemory; + private NumberFormat num; + /** * Constructor. * * @param threshold - * User specified threshold to store new values into persistent - * storage. + * User specified threshold to store new values into persistent storage. */ + public HashMapWrapper(int threshold, float loadFactor) { + this.threshold = 0.9; + mHash = new HashMap(threshold, loadFactor); + console = new LogHelper(LOG); + memoryMXBean = ManagementFactory.getMemoryMXBean(); + maxMemory = memoryMXBean.getHeapMemoryUsage().getMax(); + LOG.info("maximum memory: " + maxMemory); + num = NumberFormat.getInstance(); + num.setMinimumFractionDigits(2); + } + public HashMapWrapper(int threshold) { - this.threshold = threshold; - this.pHash = null; - this.recman = null; - this.tmpFile = null; - mHash = new HashMap(); - MRUList = new MRU(); + this(THRESHOLD, 0.75f); } public HashMapWrapper() { - this(THRESHOLD); + this(THRESHOLD, LOADFACTOR); } - /** - * Get the value based on the key. this GET method will directly - * return the value from jdbm storage. - * @param key - * @return Value corresponding to the key. If the key is not found, return - * null. - */ -/* - public V getMapJoinValueObject(K key) throws HiveException{ - if(pHash == null) { - LOG.warn("the jdbm object is not ready!"); - throw new HiveException(); - } - try{ - V value = (V)pHash.get(key); - return value; - }catch(Exception e){ - throw new HiveException(e); - } - }*/ - /* - * In this get operation, the jdbm should read only - */ - public V getMapJoinValueObject(K key) throws HiveException { - V value = null; + public V get(K key) { + return mHash.get(key); + } - // if not the MRU, searching the main memory hash table. - MRUItem item = mHash.get(key); - if (item != null) { - value = item.value; - MRUList.moveToHead(item); - } else if (pHash != null) { - try { - value = (V) pHash.get(key); - if (value != null) { - if (mHash.size() < threshold) { - MRUItem itm= new MRUItem(key, value); - mHash.put(key, itm); - //pHash.remove(key); - MRUList.put(itm); - //recman.commit(); - } else if (threshold > 0) { // flush the LRU to disk - MRUItem tail = MRUList.tail(); // least recently used item - //pHash.put(tail.key, tail.value); - //pHash.remove(key); - //recman.commit(); + public boolean put(K key, V value) throws HiveException { + // isAbort(); + mHash.put(key, value); + return false; + } - // update mHash -- reuse MRUItem - item = mHash.remove(tail.key); - item.key = key; - item.value = value; - mHash.put(key, item); + public void remove(K key) { + mHash.remove(key); + } - // update MRU -- reusing MRUItem - tail.key = key; - tail.value = value; - MRUList.moveToHead(tail); - } - } - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - } - return value; + + /** + * Flush the main memory hash table into the persistent cache file + * + * @return persistent cache file + */ + public long flushMemoryCacheToPersistent(File file) throws IOException { + ObjectOutputStream outputStream = null; + outputStream = new ObjectOutputStream(new FileOutputStream(file)); + outputStream.writeObject(mHash); + outputStream.flush(); + outputStream.close(); + + return file.length(); } - public V get(K key) throws HiveException { - V value = null; - // if not the MRU, searching the main memory hash table. - MRUItem item = mHash.get(key); - if (item != null) { - value = item.value; - MRUList.moveToHead(item); - } else if (pHash != null) { - try { - value = (V) pHash.get(key); - if (value != null) { - if (mHash.size() < threshold) { - mHash.put(key, new MRUItem(key, value)); - pHash.remove(key); - } else if (threshold > 0) { // flush the LRU to disk - MRUItem tail = MRUList.tail(); // least recently used item - pHash.put(tail.key, tail.value); - pHash.remove(key); - recman.commit(); + public void initilizePersistentHash(String fileName) throws IOException, ClassNotFoundException { + ObjectInputStream inputStream = null; + inputStream = new ObjectInputStream(new FileInputStream(fileName)); + HashMap hashtable = (HashMap) inputStream.readObject(); + this.setMHash(hashtable); - // update mHash -- reuse MRUItem - item = mHash.remove(tail.key); - item.key = key; - item.value = value; - mHash.put(key, item); + inputStream.close(); + } - // update MRU -- reusing MRUItem - tail.key = key; - tail.value = value; - MRUList.moveToHead(tail); - } - } - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - } - return value; + public int size() { + return mHash.size(); } + public Set keySet() { + return mHash.keySet(); + } + + /** - * Put the key value pair in the hash table. It will first try to put it into - * the main memory hash table. If the size exceeds the threshold, it will put - * it into the persistent hash table. + * Close the persistent hash table and clean it up. * - * @param key - * @param value * @throws HiveException */ - public void put(K key, V value) throws HiveException { - int mm_size = mHash.size(); - MRUItem itm = mHash.get(key); + public void close() throws HiveException { + // isAbort(); + mHash.clear(); + } - if (mm_size < threshold) { - if (itm != null) { - // re-use the MRU item -- just overwrite value, key is the same - itm.value = value; - MRUList.moveToHead(itm); - if (!mHash.get(key).value.equals(value)) { - LOG.error("HashMapWrapper.put() reuse MRUItem inconsistency [1]."); - } - assert (mHash.get(key).value.equals(value)); - } else { - // check if key already exists in pHash - try { - if (pHash != null && pHash.get(key) != null) { - // remove the old item from pHash and insert the new one - pHash.remove(key); - pHash.put(key, value); - recman.commit(); - return; - } - } catch (Exception e) { - e.printStackTrace(); - throw new HiveException(e); - } - itm = new MRUItem(key, value); - MRUList.put(itm); - mHash.put(key, itm); - } - } else { - if (itm != null) { // replace existing item - // re-use the MRU item -- just overwrite value, key is the same - itm.value = value; - MRUList.moveToHead(itm); - if (!mHash.get(key).value.equals(value)) { - LOG.error("HashMapWrapper.put() reuse MRUItem inconsistency [2]."); - } - assert (mHash.get(key).value.equals(value)); - } else { - // for items inserted into persistent hash table, we don't put it into - // MRU - if (pHash == null) { - pHash = getPersistentHash(); - } - try { - pHash.put(key, value); - recman.commit(); - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - } - } + public void clear() throws HiveException { + mHash.clear(); } - public void putToJDBM(K key, V value) throws HiveException{ - if (pHash == null) { - pHash = getPersistentHash(); - } - try { - pHash.put(key, value); - recman.commit(); - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } + public int getKeySize() { + return mHash.size(); + } + private boolean isAbort() { + int size = mHash.size(); + // if(size >= 1000000 && size % 1000000 == 0 ){ + System.gc(); + System.gc(); + long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed(); + double rate = (double) usedMemory / (double) maxMemory; + long mem1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + console.printInfo("Hashtable size:\t" + size + "\tMemory usage:\t" + usedMemory + "\t rate:\t" + + num.format(rate)); + return true; + } - /** - * Flush the main memory hash table into the persistent cache file - * - * @return persistent cache file - */ - public String flushMemoryCacheToPersistent() throws HiveException{ - try{ - //if no persistent cache file; create a new one - if(pHash == null){ - pHash = getPersistentHash(); - } - int mm_size = mHash.size(); - //no data in the memory cache - if(mm_size == 0){ - return tmpFile.getAbsolutePath(); - } - //iterate the memory hash table and put them into persistent file - for (Map.Entry entry : mHash.entrySet()) { - K key = entry.getKey(); - MRUItem item = entry.getValue(); - pHash.put(key, item.value); - } - //commit to the persistent file - recman.commit(); + public Log getLOG() { + return LOG; + } - //release the memory - mHash.clear(); + public void setLOG(Log log) { + LOG = log; + } - }catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - return tmpFile.getAbsolutePath(); + public double getThreshold() { + return threshold; } - public void initilizePersistentHash(File jdbmfile) throws HiveException{ - try{ - Properties props = new Properties(); - props.setProperty(RecordManagerOptions.CACHE_TYPE, - RecordManagerOptions.NORMAL_CACHE); - props.setProperty(RecordManagerOptions.DISABLE_TRANSACTIONS, "true"); + public void setThreshold(double threshold) { + this.threshold = threshold; + } - recman = RecordManagerFactory.createRecordManager(jdbmfile, props); - long recid = recman.getNamedObject( "hashtable" ); - if ( recid != 0 ) { - pHash = HTree.load( recman, recid ); - }else{ - LOG.warn("initiliaze the hash table by jdbm file Error!"); - throw new HiveException(); - } + public HashMap getMHash() { + return mHash; + } - } catch (Exception e) { - e.printStackTrace(); - LOG.warn(e.toString()); - throw new HiveException(e); - } + public void setMHash(HashMap hash) { + mHash = hash; } - /** - * Get the persistent hash table. - * - * @return persistent hash table - * @throws HiveException - */ - private HTree getPersistentHash() throws HiveException { - try { - // Create a temporary file for the page manager to hold persistent data. - if (tmpFile != null) { - tmpFile.delete(); - } - tmpFile = File.createTempFile("HashMapWrapper", ".tmp", new File("/tmp")); - LOG.info("HashMapWrapper created temp file " + tmpFile.getAbsolutePath()); - // Delete the temp file if the JVM terminate normally through Hadoop job - // kill command. - // Caveat: it won't be deleted if JVM is killed by 'kill -9'. - tmpFile.deleteOnExit(); + public LogHelper getConsole() { + return console; + } - Properties props = new Properties(); - props.setProperty(RecordManagerOptions.CACHE_TYPE, - RecordManagerOptions.NO_CACHE); - props.setProperty(RecordManagerOptions.DISABLE_TRANSACTIONS, "true"); + public void setConsole(LogHelper console) { + this.console = console; + } - recman = RecordManagerFactory.createRecordManager(tmpFile, props); - pHash = HTree.createInstance(recman); - recman.setNamedObject( "hashtable", pHash.getRecid() ); - //commit to the persistent file - recman.commit(); - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - return pHash; + public File getDumpFile() { + return dumpFile; } - /** - * Clean up the hash table. All elements in the main memory hash table will be - * removed, and the persistent hash table will be destroyed (temporary file - * will be deleted). - */ - public void clear() throws HiveException { - if (mHash != null) { - mHash.clear(); - MRUList.clear(); - } - close(); + public void setDumpFile(File dumpFile) { + this.dumpFile = dumpFile; } - /** - * Remove one key-value pairs from the hash table based on the given key. If - * the pairs are removed from the main memory hash table, pairs in the - * persistent hash table will not be moved to the main memory hash table. - * Future inserted elements will go into the main memory hash table though. - * - * @param key - * @throws HiveException - */ - public void remove(Object key) throws HiveException { - MRUItem entry = mHash.remove(key); - if (entry != null) { - MRUList.remove(entry); - } else if (pHash != null) { - try { - pHash.remove(key); - } catch (Exception e) { - LOG.warn(e.toString()); - throw new HiveException(e); - } - } + public static MemoryMXBean getMemoryMXBean() { + return memoryMXBean; } - /** - * Get a list of all keys in the hash map. - * - * @return - */ - public Set keySet() { - HashSet ret = null; - if (mHash != null) { - ret = new HashSet(); - ret.addAll(mHash.keySet()); - } - if (pHash != null) { - try { - FastIterator fitr = pHash.keys(); - if (fitr != null) { - K k; - while ((k = (K) fitr.next()) != null) { - ret.add(k); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - return ret; + public static void setMemoryMXBean(MemoryMXBean memoryMXBean) { + HashMapWrapper.memoryMXBean = memoryMXBean; } - /** - * Get the main memory cache capacity. - * - * @return the maximum number of items can be put into main memory HashMap - * cache. - */ - public int cacheSize() { - return threshold; + public long getMaxMemory() { + return maxMemory; } - /** - * Close the persistent hash table and clean it up. - * - * @throws HiveException - */ - public void close() throws HiveException { + public void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } - if (pHash != null) { - try { - if (recman != null) { - recman.close(); - } - } catch (Exception e) { - throw new HiveException(e); - } - // delete the temporary file - if(tmpFile != null){ - tmpFile.delete(); - tmpFile = null; - } - pHash = null; - recman = null; - } + public long getCurrentMemory() { + return currentMemory; } + + public void setCurrentMemory(long currentMemory) { + this.currentMemory = currentMemory; + } + + public NumberFormat getNum() { + return num; + } + + public void setNum(NumberFormat num) { + this.num = num; + } + + public static int getTHRESHOLD() { + return THRESHOLD; + } + } Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinDoubleKeys.java (revision 0) @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.io.Writable; + +public class MapJoinDoubleKeys extends AbstractMapJoinKey { + + protected transient Object obj1; + protected transient Object obj2; + + + public MapJoinDoubleKeys() { + } + + /** + * @param metadataTag + * @param obj + */ + public MapJoinDoubleKeys(Object obj1, Object obj2) { + this.obj1 = obj1; + this.obj2 = obj2; + } + + @Override + public boolean equals(Object o) { + if (o instanceof MapJoinDoubleKeys) { + MapJoinDoubleKeys mObj = (MapJoinDoubleKeys) o; + Object key1 = mObj.getObj1(); + Object key2 = mObj.getObj2(); + + if ((obj1 == null) && (key1 == null)) { + if ((obj2 == null) && (key2 == null)) { + return true; + } + } + if ((obj1 != null) && (key1 != null)) { + if (obj1.equals(key1)) { + if ((obj2 != null) && (key2 != null)) { + if (obj2.equals(key2)) { + return true; + } + } + } + } + } + return false; + } + + @Override + public int hashCode() { + int hashCode = 1; + if (obj1 == null) { + hashCode = metadataTag; + } else { + hashCode += (31 + obj1.hashCode()); + } + if (obj2 == null) { + hashCode += metadataTag; + } else { + hashCode += (31 + obj2.hashCode()); + } + return hashCode; + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + try { + // get the tableDesc from the map stored in the mapjoin operator + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); + + Writable val = ctx.getSerDe().getSerializedClass().newInstance(); + val.readFields(in); + + + + ArrayList list = (ArrayList) ObjectInspectorUtils.copyToStandardObject(ctx + .getSerDe().deserialize(val), ctx.getSerDe().getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE); + + if (list == null) { + obj1 = null; + obj2 = null; + + } else { + obj1 = list.get(0); + obj2 = list.get(1); + } + + } catch (Exception e) { + throw new IOException(e); + } + + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + try { + // out.writeInt(metadataTag); + // get the tableDesc from the map stored in the mapjoin operator + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); + + ArrayList list = MapJoinMetaData.getList(); + list.add(obj1); + list.add(obj2); + // Different processing for key and value + Writable outVal = ctx.getSerDe().serialize(list, ctx.getStandardOI()); + outVal.write(out); + + } catch (SerDeException e) { + throw new IOException(e); + } + } + + + + /** + * @return the obj + */ + public Object getObj1() { + return obj1; + } + + /** + * @param obj + * the obj to set + */ + public void setObj1(Object obj1) { + this.obj1 = obj1; + } + + /** + * @return the obj + */ + public Object getObj2() { + return obj2; + } + + /** + * @param obj + * the obj to set + */ + public void setObj2(Object obj2) { + this.obj2 = obj2; + } + + + @Override + public boolean hasAnyNulls() { + if (obj1 == null) { + return true; + } + if (obj2 == null) { + return true; + } + return false; + } +} Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java (revision 0) @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.metadata.HiveException; + +public class MapJoinRowContainer extends AbstractRowContainer { + + private List list; + + private int index; + + public MapJoinRowContainer() { + index = 0; + list = new ArrayList(1); + } + + @Override + public void add(Row t) throws HiveException { + list.add(t); + } + + + @Override + public Row first() throws HiveException { + index = 0; + if (index < list.size()) { + return list.get(index); + } + return null; + } + + @Override + public Row next() throws HiveException { + index++; + if (index < list.size()) { + return list.get(index); + } + return null; + + } + + /** + * Get the number of elements in the RowContainer. + * + * @return number of elements in the RowContainer + */ + @Override + public int size() { + return list.size(); + } + + /** + * Remove all elements in the RowContainer. + */ + @Override + public void clear() throws HiveException { + list.clear(); + index = 0; + } + + public List getList() { + return list; + } + + public void setList(List list) { + this.list = list; + } + + public void reset(MapJoinRowContainer other) throws HiveException { + list.clear(); + Object[] obj; + for (obj = other.first(); obj != null; obj = other.next()) { + ArrayList ele = new ArrayList(obj.length); + for (int i = 0; i < obj.length; i++) { + ele.add(obj[i]); + } + list.add((Row) ele); + } + } +} Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinSingleKey.java (revision 0) @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.ArrayList; + +import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; +import org.apache.hadoop.io.Writable; + +public class MapJoinSingleKey extends AbstractMapJoinKey { + + + protected transient Object obj; + + public MapJoinSingleKey() { + } + + /** + * @param metadataTag + * @param obj + */ + public MapJoinSingleKey(Object obj) { + this.obj = obj; + } + + @Override + public boolean equals(Object o) { + if (o instanceof MapJoinSingleKey) { + MapJoinSingleKey mObj = (MapJoinSingleKey) o; + Object key = mObj.getObj(); + if ((obj == null) && (key == null)) { + return true; + } + if ((obj != null) && (key != null)) { + if (obj.equals(key)) { + return true; + } + } + } + return false; + } + + @Override + public int hashCode() { + int hashCode; + if (obj == null) { + hashCode = metadataTag; + } else { + hashCode = 31 + obj.hashCode(); + } + return hashCode; + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + try { + // get the tableDesc from the map stored in the mapjoin operator + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); + + Writable val = ctx.getSerDe().getSerializedClass().newInstance(); + val.readFields(in); + + + + ArrayList list = (ArrayList) ObjectInspectorUtils.copyToStandardObject(ctx + .getSerDe().deserialize(val), ctx.getSerDe().getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE); + + if (list == null) { + obj = null; + System.out.println("read empty back"); + } else { + obj = list.get(0); + } + + } catch (Exception e) { + throw new IOException(e); + } + + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + try { + // out.writeInt(metadataTag); + // get the tableDesc from the map stored in the mapjoin operator + HashTableSinkObjectCtx ctx = MapJoinMetaData.get(Integer.valueOf(metadataTag)); + + ArrayList list = MapJoinMetaData.getList(); + list.add(obj); + + // Different processing for key and value + Writable outVal = ctx.getSerDe().serialize(list, ctx.getStandardOI()); + outVal.write(out); + + } catch (SerDeException e) { + throw new IOException(e); + } + } + + + + /** + * @return the obj + */ + public Object getObj() { + return obj; + } + + /** + * @param obj + * the obj to set + */ + public void setObj(Object obj) { + this.obj = obj; + } + + @Override + public boolean hasAnyNulls() { + if (obj == null) { + return true; + } + return false; + } + + + +} Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (working copy) @@ -51,31 +51,27 @@ /** * Simple persistent container for rows. * - * This container interface only accepts adding or appending new rows and - * iterating through the rows in the order of their insertions. + * This container interface only accepts adding or appending new rows and iterating through the rows + * in the order of their insertions. * - * The iterator interface is a lightweight first()/next() API rather than the - * Java Iterator interface. This way we do not need to create an Iterator object - * every time we want to start a new iteration. Below is simple example of how - * to convert a typical Java's Iterator code to the LW iterator interface. + * The iterator interface is a lightweight first()/next() API rather than the Java Iterator + * interface. This way we do not need to create an Iterator object every time we want to start a new + * iteration. Below is simple example of how to convert a typical Java's Iterator code to the LW + * iterator interface. * - * Iterator itr = rowContainer.iterator(); - * while (itr.hasNext()) { - * v = itr.next(); // do anything with v - * } + * Iterator itr = rowContainer.iterator(); while (itr.hasNext()) { v = itr.next(); // do anything + * with v } * * can be rewritten to: * - * for ( v = rowContainer.first(); v != null; v = rowContainer.next()) { - * // do anything with v - * } + * for ( v = rowContainer.first(); v != null; v = rowContainer.next()) { // do anything with v } * - * Once the first is called, it will not be able to write again. So there can - * not be any writes after read. It can be read multiple times, but it does not - * support multiple reader interleaving reading. + * Once the first is called, it will not be able to write again. So there can not be any writes + * after read. It can be read multiple times, but it does not support multiple reader interleaving + * reading. * */ -public class RowContainer> { +public class RowContainer> extends AbstractRowContainer { protected static Log LOG = LogFactory.getLog(RowContainer.class); @@ -120,6 +116,10 @@ JobConf jobCloneUsingLocalFs = null; private LocalFileSystem localFs; + public RowContainer() { + + } + public RowContainer(Configuration jc) throws HiveException { this(BLOCKSIZE, jc); } @@ -137,21 +137,20 @@ this.firstReadBlockPointer = currentReadBlock; this.serde = null; this.standardOI = null; - this.jc=jc; + this.jc = jc; } private JobConf getLocalFSJobConfClone(Configuration jc) { - if(this.jobCloneUsingLocalFs == null) { + if (this.jobCloneUsingLocalFs == null) { this.jobCloneUsingLocalFs = new JobConf(jc); - HiveConf.setVar(jobCloneUsingLocalFs, HiveConf.ConfVars.HADOOPFS, - Utilities.HADOOP_LOCAL_FS); + HiveConf.setVar(jobCloneUsingLocalFs, HiveConf.ConfVars.HADOOPFS, Utilities.HADOOP_LOCAL_FS); } return this.jobCloneUsingLocalFs; } - public RowContainer(int blockSize, SerDe sd, ObjectInspector oi, - Configuration jc) throws HiveException { + public RowContainer(int blockSize, SerDe sd, ObjectInspector oi, Configuration jc) + throws HiveException { this(blockSize, jc); setSerDe(sd, oi); } @@ -161,6 +160,7 @@ this.standardOI = oi; } + @Override public void add(Row t) throws HiveException { if (this.tblDesc != null) { if (addCursor >= blockSize) { // spill the current block to tmp file @@ -180,6 +180,7 @@ ++size; } + @Override public Row first() throws HiveException { if (size == 0) { return null; @@ -209,21 +210,17 @@ JobConf localJc = getLocalFSJobConfClone(jc); if (inputSplits == null) { if (this.inputFormat == null) { - inputFormat = (InputFormat) ReflectionUtils - .newInstance(tblDesc.getInputFileFormatClass(), - localJc); + inputFormat = (InputFormat) ReflectionUtils.newInstance( + tblDesc.getInputFileFormatClass(), localJc); } - HiveConf.setVar(localJc, - HiveConf.ConfVars.HADOOPMAPREDINPUTDIR, - org.apache.hadoop.util.StringUtils.escapeString(parentFile - .getAbsolutePath())); + HiveConf.setVar(localJc, HiveConf.ConfVars.HADOOPMAPREDINPUTDIR, + org.apache.hadoop.util.StringUtils.escapeString(parentFile.getAbsolutePath())); inputSplits = inputFormat.getSplits(localJc, 1); acutalSplitNum = inputSplits.length; } currentSplitPointer = 0; - rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], - localJc, Reporter.NULL); + rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], localJc, Reporter.NULL); currentSplitPointer++; nextBlock(); @@ -238,6 +235,7 @@ } + @Override public Row next() throws HiveException { if (!firstCalled) { @@ -278,8 +276,7 @@ } private void removeKeys(Row ret) { - if (this.keyObject != null - && this.currentReadBlock != this.currentWriteBlock) { + if (this.keyObject != null && this.currentReadBlock != this.currentWriteBlock) { int len = this.keyObject.size(); int rowSize = ((ArrayList) ret).size(); for (int i = 0; i < len; i++) { @@ -317,16 +314,13 @@ tmpFile.deleteOnExit(); // rFile = new RandomAccessFile(tmpFile, "rw"); - HiveOutputFormat hiveOutputFormat = tblDesc - .getOutputFileFormatClass().newInstance(); + HiveOutputFormat hiveOutputFormat = tblDesc.getOutputFileFormatClass().newInstance(); tempOutPath = new Path(tmpFile.toString()); JobConf localJc = getLocalFSJobConfClone(jc); - rw = HiveFileFormatUtils.getRecordWriter(this.jobCloneUsingLocalFs, - hiveOutputFormat, serde.getSerializedClass(), false, tblDesc - .getProperties(), tempOutPath); + rw = HiveFileFormatUtils.getRecordWriter(this.jobCloneUsingLocalFs, hiveOutputFormat, serde + .getSerializedClass(), false, tblDesc.getProperties(), tempOutPath); } else if (rw == null) { - throw new HiveException( - "RowContainer has already been closed for writing."); + throw new HiveException("RowContainer has already been closed for writing."); } row.clear(); @@ -366,6 +360,7 @@ * * @return number of elements in the RowContainer */ + @Override public int size() { return size; } @@ -388,17 +383,16 @@ Object key = rr.createKey(); while (i < this.currentReadBlock.length && rr.next(key, val)) { nextSplit = false; - this.currentReadBlock[i++] = (Row) ObjectInspectorUtils - .copyToStandardObject(serde.deserialize(val), serde - .getObjectInspector(), ObjectInspectorCopyOption.WRITABLE); + this.currentReadBlock[i++] = (Row) ObjectInspectorUtils.copyToStandardObject(serde + .deserialize(val), serde.getObjectInspector(), ObjectInspectorCopyOption.WRITABLE); } } if (nextSplit && this.currentSplitPointer < this.acutalSplitNum) { JobConf localJc = getLocalFSJobConfClone(jc); // open record reader to read next split - rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], - jobCloneUsingLocalFs, Reporter.NULL); + rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], jobCloneUsingLocalFs, + Reporter.NULL); currentSplitPointer++; return nextBlock(); } @@ -416,8 +410,7 @@ } } - public void copyToDFSDirecory(FileSystem destFs, Path destPath) - throws IOException, HiveException { + public void copyToDFSDirecory(FileSystem destFs, Path destPath) throws IOException, HiveException { if (addCursor > 0) { this.spillBlock(this.currentWriteBlock, addCursor); } @@ -425,16 +418,17 @@ return; } this.closeWriter(); - LOG.info("RowContainer copied temp file " + tmpFile.getAbsolutePath() - + " to dfs directory " + destPath.toString()); - destFs.copyFromLocalFile(true, tempOutPath, new Path(destPath, new Path( - tempOutPath.getName()))); + LOG.info("RowContainer copied temp file " + tmpFile.getAbsolutePath() + " to dfs directory " + + destPath.toString()); + destFs + .copyFromLocalFile(true, tempOutPath, new Path(destPath, new Path(tempOutPath.getName()))); clear(); } /** * Remove all elements in the RowContainer. */ + @Override public void clear() throws HiveException { itrCursor = 0; addCursor = 0; Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinKey.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinKey.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractMapJoinKey.java (revision 0) @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; + +/** + * Map Join Object used for both key. + */ +public abstract class AbstractMapJoinKey implements Externalizable { + + protected static int metadataTag = -1; + + public AbstractMapJoinKey() { + } + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract int hashCode(); + + public abstract void readExternal(ObjectInput in) throws IOException, ClassNotFoundException; + + public abstract void writeExternal(ObjectOutput out) throws IOException; + + public abstract boolean hasAnyNulls(); + +} Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectKey.java (working copy) @@ -18,14 +18,13 @@ package org.apache.hadoop.hive.ql.exec.persistence; -import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.ArrayList; import org.apache.hadoop.hive.ql.exec.MapJoinMetaData; -import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; @@ -34,11 +33,11 @@ /** * Map Join Object used for both key. */ -public class MapJoinObjectKey implements Externalizable { +public class MapJoinObjectKey extends AbstractMapJoinKey { - protected transient int metadataTag; - protected transient ArrayList obj; + protected transient Object[] obj; + public MapJoinObjectKey() { } @@ -46,8 +45,7 @@ * @param metadataTag * @param obj */ - public MapJoinObjectKey(int metadataTag, ArrayList obj) { - this.metadataTag = metadataTag; + public MapJoinObjectKey(Object[] obj) { this.obj = obj; } @@ -55,43 +53,60 @@ public boolean equals(Object o) { if (o instanceof MapJoinObjectKey) { MapJoinObjectKey mObj = (MapJoinObjectKey) o; - if (mObj.getMetadataTag() == metadataTag) { - if ((obj == null) && (mObj.getObj() == null)) { + Object[] mObjArray = mObj.getObj(); + if ((obj == null) && (mObjArray == null)) { + return true; + } + if ((obj != null) && (mObjArray != null)){ + if(obj.length == mObjArray.length){ + for(int i = 0; i) ObjectInspectorUtils.copyToStandardObject(ctx + ArrayList list = (ArrayList) ObjectInspectorUtils.copyToStandardObject(ctx .getSerDe().deserialize(val), ctx.getSerDe().getObjectInspector(), ObjectInspectorCopyOption.WRITABLE); - if(obj == null){ - obj = new ArrayList(0); + if(list == null){ + obj = new ArrayList(0).toArray(); + }else{ + obj = list.toArray(); } + } catch (Exception e) { throw new IOException(e); } @@ -101,9 +116,9 @@ @Override public void writeExternal(ObjectOutput out) throws IOException { try { - out.writeInt(metadataTag); + //out.writeInt(metadataTag); // get the tableDesc from the map stored in the mapjoin operator - JDBMSinkObjectCtx ctx = MapJoinMetaData.get( + HashTableSinkObjectCtx ctx = MapJoinMetaData.get( Integer.valueOf(metadataTag)); // Different processing for key and value @@ -114,25 +129,12 @@ } } - /** - * @return the metadataTag - */ - public int getMetadataTag() { - return metadataTag; - } - /** - * @param metadataTag - * the metadataTag to set - */ - public void setMetadataTag(int metadataTag) { - this.metadataTag = metadataTag; - } /** * @return the obj */ - public ArrayList getObj() { + public Object[] getObj() { return obj; } @@ -140,8 +142,21 @@ * @param obj * the obj to set */ - public void setObj(ArrayList obj) { + public void setObj(Object[] obj) { this.obj = obj; } + @Override + public boolean hasAnyNulls(){ + if (obj != null && obj.length> 0) { + for (Object k : obj) { + if (k == null) { + return true; + } + } + } + return false; + + } + } Index: src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java (revision 0) @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.persistence; + +import org.apache.hadoop.hive.ql.metadata.HiveException; + +public abstract class AbstractRowContainer { + + public AbstractRowContainer() { + + } + + public abstract void add(Row t) throws HiveException; + + public abstract Row first() throws HiveException; + + public abstract Row next() throws HiveException; + + /** + * Get the number of elements in the RowContainer. + * + * @return number of elements in the RowContainer + */ + + public abstract int size(); + + /** + * Remove all elements in the RowContainer. + */ + + public abstract void clear() throws HiveException; +} Index: src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (working copy) @@ -18,26 +18,39 @@ package org.apache.hadoop.hive.ql.exec; import java.io.File; +import java.io.IOException; +import java.io.OutputStream; import java.io.Serializable; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.MapredLocalWork.BucketMapJoinContext; import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -47,11 +60,14 @@ public class MapredLocalTask extends Task implements Serializable { private Map fetchOperators; - private File jdbmFile; private JobConf job; public static final Log l4j = LogFactory.getLog("MapredLocalTask"); - private MapOperator mo; - // not sure we need this exec context; but all the operators in the work + static final String HADOOP_MEM_KEY = "HADOOP_HEAPSIZE"; + static final String HADOOP_OPTS_KEY = "HADOOP_OPTS"; + static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive"}; + public static MemoryMXBean memoryMXBean; + + // not sure we need this exec context; but all the operators in the work // will pass this context throught private final ExecMapperContext execContext = new ExecMapperContext(); @@ -59,6 +75,13 @@ super(); } + public MapredLocalTask(MapredLocalWork plan, JobConf job, boolean isSilent) throws HiveException { + setWork(plan); + this.job = job; + LOG = LogFactory.getLog(this.getClass().getName()); + console = new LogHelper(LOG, isSilent); + } + @Override public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext driverContext) { @@ -66,12 +89,159 @@ job = new JobConf(conf, ExecDriver.class); } + public static String now(){ + Calendar cal = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd hh:mm:ss"); + return sdf.format(cal.getTime()); + } + + + @Override -public int execute(DriverContext driverContext){ + public int execute(DriverContext driverContext){ + try{ + //generate the cmd line to run in the child jvm + //String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN); + Context ctx = driverContext.getCtx(); + String hiveJar = conf.getJar(); + + String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN); + String libJarsOption; + + // write out the plan to a local file + Path planPath = new Path(ctx.getLocalTmpFileURI(), "plan.xml"); + OutputStream out = FileSystem.getLocal(conf).create(planPath); + MapredLocalWork plan = getWork(); + LOG.info("Generating plan file " + planPath.toString()); + Utilities.serializeMapRedLocalWork(plan, out); + + String isSilent = "true".equalsIgnoreCase(System + .getProperty("test.silent")) ? "-nolog" : ""; + + String jarCmd; + + jarCmd = hiveJar + " " + ExecDriver.class.getName() ; + + String hiveConfArgs = ExecDriver.generateCmdLine(conf); + String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + + planPath.toString() + " " + isSilent + " " + hiveConfArgs; + + String workDir = (new File(".")).getCanonicalPath(); + String files = ExecDriver.getResourceFiles(conf, SessionState.ResourceType.FILE); + + if (!files.isEmpty()) { + cmdLine = cmdLine + " -files " + files; + + workDir = (new Path(ctx.getLocalTmpFileURI())).toUri().getPath(); + + if (! (new File(workDir)).mkdir()) { + throw new IOException ("Cannot create tmp working dir: " + workDir); + } + + for (String f: StringUtils.split(files, ',')) { + Path p = new Path(f); + String target = p.toUri().getPath(); + String link = workDir + Path.SEPARATOR + p.getName(); + if (FileUtil.symLink(target, link) != 0) { + throw new IOException ("Cannot link to added file: " + target + " from: " + link); + } + } + } + + LOG.info("Executing: " + cmdLine); + Process executor = null; + + // Inherit Java system variables + String hadoopOpts; + StringBuilder sb = new StringBuilder(); + Properties p = System.getProperties(); + for (String element : HIVE_SYS_PROP) { + if (p.containsKey(element)) { + sb.append(" -D" + element + "=" + p.getProperty(element)); + } + } + hadoopOpts = sb.toString(); + // Inherit the environment variables + String[] env; + Map variables = new HashMap(System.getenv()); + // The user can specify the hadoop memory + + //if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) { + // if we are running in local mode - then the amount of memory used + // by the child jvm can no longer default to the memory used by the + // parent jvm + //int hadoopMem = conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM); + int hadoopMem= conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);; + if (hadoopMem == 0) { + // remove env var that would default child jvm to use parent's memory + // as default. child jvm would use default memory for a hadoop client + variables.remove(HADOOP_MEM_KEY); + } else { + // user specified the memory for local mode hadoop run + console.printInfo(" set heap size\t"+hadoopMem+"MB"); + variables.put(HADOOP_MEM_KEY, String.valueOf(hadoopMem)); + } + //} else { + // nothing to do - we are not running in local mode - only submitting + // the job via a child process. in this case it's appropriate that the + // child jvm use the same memory as the parent jvm + + //} + + if (variables.containsKey(HADOOP_OPTS_KEY)) { + variables.put(HADOOP_OPTS_KEY, variables.get(HADOOP_OPTS_KEY) + + hadoopOpts); + } else { + variables.put(HADOOP_OPTS_KEY, hadoopOpts); + } + env = new String[variables.size()]; + int pos = 0; + for (Map.Entry entry : variables.entrySet()) { + String name = entry.getKey(); + String value = entry.getValue(); + env[pos++] = name + "=" + value; + } + + // Run ExecDriver in another JVM + executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir)); + + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, System.err); + + outPrinter.start(); + errPrinter.start(); + + int exitVal = executor.waitFor(); + + if (exitVal != 0) { + LOG.error("Execution failed with exit status: " + exitVal); + console.printError("Mapred Local Task Failed. Give up the map join stragery"); + } else { + LOG.info("Execution completed successfully"); + console.printInfo("Mapred Local Task Running Successfully . Keep using map join stragery"); + } + + return exitVal; + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Exception: " + e.getMessage()); + return (1); + } + } + + + + public int executeFromChildJVM(DriverContext driverContext){ + // check the local work if(work == null){ return -1; } + memoryMXBean = ManagementFactory.getMemoryMXBean(); + console.printInfo(Utilities.now()+"\tStarting to luaunch local task to process map join "); + console.printInfo("\tmaximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); fetchOperators = new HashMap(); Map fetchOpJobConfMap = new HashMap(); execContext.setJc(job); @@ -92,14 +262,19 @@ }else{ startForward(inputFileChangeSenstive,null); } + console.printInfo(now()+"\tEnd of local task "); } catch (Throwable e) { if (e instanceof OutOfMemoryError) { // Don't create a new object if we are already out of memory - l4j.error("Out of Merror Error"); + l4j.error("Out of Memory Error"); + console.printError("[Warning] Small table is too large to put into memory"); + return 2; } else { l4j.error("Hive Runtime Error: Map local work failed"); e.printStackTrace(); } + }finally{ + console.printInfo(Utilities.now()+"\tFinish running local task"); } return 0; } Index: src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (working copy) @@ -17,9 +17,8 @@ */ package org.apache.hadoop.hive.ql.exec; -import java.io.File; + import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -30,15 +29,15 @@ import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; +import org.apache.hadoop.hive.ql.exec.persistence.AbstractMapJoinKey; import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectKey; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectValue; +import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -48,28 +47,24 @@ /** * Map side Join operator implementation. */ -public class MapJoinOperator extends AbstractMapJoinOperator implements - Serializable { +public class MapJoinOperator extends AbstractMapJoinOperator implements Serializable { private static final long serialVersionUID = 1L; - private static final Log LOG = LogFactory.getLog(MapJoinOperator.class - .getName()); + private static final Log LOG = LogFactory.getLog(MapJoinOperator.class.getName()); - protected transient Map> mapJoinTables; + protected transient Map> mapJoinTables; private static final transient String[] FATAL_ERR_MSG = { null, // counter value 0 means no error "Mapside join size exceeds hive.mapjoin.maxsize. " - + "Please increase that or remove the mapjoin hint." - }; + + "Please increase that or remove the mapjoin hint."}; - - - + protected transient Map>> rowContainerMap; transient int metadataKeyTag; transient int[] metadataValueTag; transient int maxMapJoinSize; private int bigTableAlias; + public MapJoinOperator() { } @@ -82,8 +77,7 @@ super.initializeOp(hconf); - maxMapJoinSize = HiveConf.getIntVar(hconf, - HiveConf.ConfVars.HIVEMAXMAPJOINSIZE); + maxMapJoinSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAXMAPJOINSIZE); metadataValueTag = new int[numAliases]; for (int pos = 0; pos < numAliases; pos++) { @@ -93,22 +87,22 @@ metadataKeyTag = -1; bigTableAlias = order[posBigTable]; - mapJoinTables = new HashMap>(); - + mapJoinTables = new HashMap>(); + rowContainerMap = new HashMap>>(); // initialize the hash tables for other tables for (int pos = 0; pos < numAliases; pos++) { if (pos == posBigTable) { continue; } - int cacheSize = HiveConf.getIntVar(hconf, - HiveConf.ConfVars.HIVEMAPJOINCACHEROWS); - HashMapWrapper hashTable = new HashMapWrapper( - cacheSize); + HashMapWrapper hashTable = new HashMapWrapper(); mapJoinTables.put(Byte.valueOf((byte) pos), hashTable); + MapJoinRowContainer> rowContainer = new MapJoinRowContainer>(); + rowContainerMap.put(Byte.valueOf((byte) pos), rowContainer); } + } @@ -119,112 +113,101 @@ } - public void generateMapMetaData() throws HiveException,SerDeException{ - //generate the meta data for key - //index for key is -1 + public void generateMapMetaData() throws HiveException, SerDeException { + // generate the meta data for key + // index for key is -1 TableDesc keyTableDesc = conf.getKeyTblDesc(); - SerDe keySerializer = (SerDe) ReflectionUtils.newInstance( - keyTableDesc.getDeserializerClass(), null); + SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), + null); keySerializer.initialize(null, keyTableDesc.getProperties()); - MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), - new JDBMSinkObjectCtx( - ObjectInspectorUtils - .getStandardObjectInspector(keySerializer - .getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), keySerializer, - keyTableDesc, hconf)); + MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx( + ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(), + ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf)); - //index for values is just alias + // index for values is just alias for (int tag = 0; tag < order.length; tag++) { int alias = (int) order[tag]; - if(alias == this.bigTableAlias){ + if (alias == this.bigTableAlias) { continue; } TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); - SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc - .getDeserializerClass(), null); + SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), + null); valueSerDe.initialize(null, valueTableDesc.getProperties()); - MapJoinMetaData.put(Integer.valueOf(alias), - new JDBMSinkObjectCtx(ObjectInspectorUtils + MapJoinMetaData.put(Integer.valueOf(alias), new HashTableSinkObjectCtx(ObjectInspectorUtils .getStandardObjectInspector(valueSerDe.getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), valueSerDe, - valueTableDesc, hconf)); + ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf)); } } - private void loadJDBM() throws HiveException{ + private void loadHashTable() throws HiveException { boolean localMode = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPJT).equals("local"); - String tmpURI =null; - HashMapWrapper hashtable; + String tmpURI = null; + HashMapWrapper hashtable; Byte pos; int alias; - String currentInputFile = HiveConf.getVar(hconf, - HiveConf.ConfVars.HADOOPMAPFILENAME); + String currentInputFile = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME); + LOG.info("******* Load from HashTable File: input : " + currentInputFile); String currentFileName; - if(this.getExecContext().getLocalWork().getInputFileChangeSensitive()) { - currentFileName= this.getFileName(currentInputFile); + if (this.getExecContext().getLocalWork().getInputFileChangeSensitive()) { + currentFileName = this.getFileName(currentInputFile); } else { - currentFileName="-"; + currentFileName = "-"; } - LOG.info("******* Filename : "+ currentFileName); - try{ - if(localMode){ - //load the jdbm file from tmp dir - tmpURI= this.getExecContext().getLocalWork().getTmpFileURI(); - for(Map.Entry> entry: mapJoinTables.entrySet()){ + + try { + if (localMode) { + LOG.info("******* Load from tmp file uri ***"); + tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); + for (Map.Entry> entry : mapJoinTables + .entrySet()) { pos = entry.getKey(); - hashtable=entry.getValue(); - URI uri = new URI(tmpURI+Path.SEPARATOR+"-"+pos+"-"+currentFileName+".jdbm"); - LOG.info("\tLoad back 1 JDBM file from tmp file uri:"+uri.toString()); - Path path = new Path(tmpURI+Path.SEPARATOR+"-"+pos+"-"+currentFileName+".jdbm"); - LOG.info("\tLoad back 1 JDBM file from tmp file uri:"+path.toString()); + hashtable = entry.getValue(); + String filePath = Utilities.generatePath(tmpURI, pos, currentFileName); + Path path = new Path(filePath); + LOG.info("\tLoad back 1 hashtable file from tmp file uri:" + path.toString()); - File jdbmFile = new File(path.toUri()); - hashtable.initilizePersistentHash(jdbmFile); + hashtable.initilizePersistentHash(path.toUri().getPath()); } - }else{ - //load the jdbm file from distributed cache - Path[] localFiles= DistributedCache.getLocalCacheFiles(this.hconf); - for(int i = 0;i> entry: mapJoinTables.entrySet()){ - pos = entry.getKey(); - hashtable=entry.getValue(); - String suffix="-"+pos+"-"+currentFileName+".jdbm"; - LOG.info("Looking for jdbm file with suffix: "+suffix); + for (Map.Entry> entry : mapJoinTables + .entrySet()) { + pos = entry.getKey(); + hashtable = entry.getValue(); + String suffix = Utilities.generateFileName(pos, currentFileName); + LOG.info("Looking for hashtable file with suffix: " + suffix); - boolean found=false; - for(int i = 0;i key = JoinUtil.computeKeys(row, joinKeys.get(alias), + AbstractMapJoinKey key = JoinUtil.computeMapJoinKeys(row, joinKeys.get(alias), joinKeysObjectInspectors.get(alias)); ArrayList value = JoinUtil.computeValues(row, joinValues.get(alias), - joinValuesObjectInspectors.get(alias), joinFilters.get(alias), - joinFilterObjectInspectors.get(alias), noOuterJoin); + joinValuesObjectInspectors.get(alias), joinFilters.get(alias), joinFilterObjectInspectors + .get(alias), noOuterJoin); // Add the value to the ArrayList - storage.get((byte)tag).add(value); + storage.get((byte) tag).add(value); for (Byte pos : order) { if (pos.intValue() != tag) { - MapJoinObjectKey keyMap = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue o = mapJoinTables.get(pos).getMapJoinValueObject(keyMap); + MapJoinObjectValue o = mapJoinTables.get(pos).get(key); + MapJoinRowContainer> rowContainer = rowContainerMap.get(pos); + // there is no join-value or join-key has all null elements - if (o == null || (hasAnyNulls(key))) { + if (o == null || key.hasAnyNulls()) { if (noOuterJoin) { storage.put(pos, emptyList); } else { storage.put(pos, dummyObjVectors[pos.intValue()]); } } else { - storage.put(pos, o.getObj()); + rowContainer.reset(o.getObj()); + storage.put(pos, rowContainer); } } } @@ -288,7 +273,7 @@ checkAndGenObject(); // done with the row - storage.get((byte)tag).clear(); + storage.get((byte) tag).clear(); for (Byte pos : order) { if (pos.intValue() != tag) { @@ -301,20 +286,22 @@ throw new HiveException(e); } } - private String getFileName(String path){ - if(path== null || path.length()==0) { + + private String getFileName(String path) { + if (path == null || path.length() == 0) { return null; } - int last_separator = path.lastIndexOf(Path.SEPARATOR)+1; + int last_separator = path.lastIndexOf(Path.SEPARATOR) + 1; String fileName = path.substring(last_separator); return fileName; } + @Override public void closeOp(boolean abort) throws HiveException { - if(mapJoinTables != null) { + if (mapJoinTables != null) { for (HashMapWrapper hashTable : mapJoinTables.values()) { hashTable.close(); } Index: src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java (working copy) @@ -28,8 +28,8 @@ import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; -import org.apache.hadoop.hive.ql.plan.JDBMDummyDesc; -import org.apache.hadoop.hive.ql.plan.JDBMSinkDesc; +import org.apache.hadoop.hive.ql.plan.HashTableDummyDesc; +import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.LateralViewForwardDesc; import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc; @@ -87,10 +87,10 @@ LateralViewJoinOperator.class)); opvec.add(new OpTuple(LateralViewForwardDesc.class, LateralViewForwardOperator.class)); - opvec.add(new OpTuple(JDBMDummyDesc.class, - JDBMDummyOperator.class)); - opvec.add(new OpTuple(JDBMSinkDesc.class, - JDBMSinkOperator.class)); + opvec.add(new OpTuple(HashTableDummyDesc.class, + HashTableDummyOperator.class)); + opvec.add(new OpTuple(HashTableSinkDesc.class, + HashTableSinkOperator.class)); } public static Operator get(Class opClass) { Index: src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -23,6 +23,8 @@ import java.io.InputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; @@ -101,7 +103,7 @@ protected transient JobConf job; protected transient int mapProgress = 0; protected transient int reduceProgress = 0; - + public static MemoryMXBean memoryMXBean; /** * Constructor when invoked from QL. */ @@ -597,35 +599,37 @@ if(localwork != null){ boolean localMode = HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local"); if(!localMode){ - LOG.info("=========Adding files to HDFS ================"); + //LOG.info("=========Adding files to HDFS ================"); Path localPath = new Path(localwork.getTmpFileURI()); Path hdfsPath = new Path(work.getTmpHDFSFileURI()); - FileSystem hdfs = hdfsPath.getFileSystem(job); + FileSystem hdfs = hdfsPath.getFileSystem(job); FileSystem localFS = localPath.getFileSystem(job); - FileStatus[] jdbmFiles = localFS.listStatus(localPath); - for(int i =0; i jobConfArgs = new ArrayList(); boolean noLog = false; String files = null; + boolean localtask=false; try { for (int i = 0; i < args.length; i++) { @@ -981,6 +986,8 @@ noLog = true; } else if (args[i].equals("-files")) { files = args[++i]; + }else if(args[i].equals("-localtask")) { + localtask = true; } } } catch (IndexOutOfBoundsException e) { @@ -988,7 +995,12 @@ printUsage(); } - JobConf conf = new JobConf(ExecDriver.class); + JobConf conf ; + if(localtask) { + conf= new JobConf(MapredLocalTask.class); + } else { + conf= new JobConf(ExecDriver.class); + } StringBuilder sb = new StringBuilder("JobConf:\n"); for (String one : jobConfArgs) { @@ -1071,12 +1083,19 @@ } catch (Exception e) { throw new HiveException(e.getMessage(), e); } + int ret; + if(localtask) { + memoryMXBean = ManagementFactory.getMemoryMXBean(); + MapredLocalWork plan = Utilities.deserializeMapRedLocalWork(pathData, conf); + MapredLocalTask ed = new MapredLocalTask(plan, conf, isSilent); + ret= ed.executeFromChildJVM(new DriverContext()); + } else { + MapredWork plan = Utilities.deserializeMapRedWork(pathData, conf); + ExecDriver ed = new ExecDriver(plan, conf, isSilent); + ret = ed.execute(new DriverContext()); + } - MapredWork plan = Utilities.deserializeMapRedWork(pathData, conf); - ExecDriver ed = new ExecDriver(plan, conf, isSilent); - - int ret = ed.execute(new DriverContext()); if (ret != 0) { System.exit(2); } Index: src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -43,8 +43,10 @@ import java.net.URI; import java.net.URL; import java.net.URLClassLoader; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -88,11 +90,12 @@ import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.stats.StatsFactory; import org.apache.hadoop.hive.ql.stats.StatsPublisher; import org.apache.hadoop.hive.serde.Constants; @@ -193,14 +196,13 @@ } /** - * Java 1.5 workaround. From - * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=5015403 + * Java 1.5 workaround. From http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=5015403 */ public static class EnumDelegate extends DefaultPersistenceDelegate { @Override protected Expression instantiate(Object oldInstance, Encoder out) { - return new Expression(Enum.class, "valueOf", new Object[] { - oldInstance.getClass(), ((Enum) oldInstance).name()}); + return new Expression(Enum.class, "valueOf", new Object[] {oldInstance.getClass(), + ((Enum) oldInstance).name()}); } @Override @@ -212,24 +214,26 @@ public static class MapDelegate extends DefaultPersistenceDelegate { @Override protected Expression instantiate(Object oldInstance, Encoder out) { - Map oldMap = (Map)oldInstance; + Map oldMap = (Map) oldInstance; HashMap newMap = new HashMap(oldMap); return new Expression(newMap, HashMap.class, "new", new Object[] {}); } + @Override protected boolean mutatesTo(Object oldInstance, Object newInstance) { return false; } + @Override protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) { - java.util.Collection oldO = (java.util.Collection)oldInstance; - java.util.Collection newO = (java.util.Collection)newInstance; + java.util.Collection oldO = (java.util.Collection) oldInstance; + java.util.Collection newO = (java.util.Collection) newInstance; if (newO.size() != 0) { - out.writeStatement(new Statement(oldInstance, "clear", new Object[]{})); + out.writeStatement(new Statement(oldInstance, "clear", new Object[] {})); } for (Iterator i = oldO.iterator(); i.hasNext();) { - out.writeStatement(new Statement(oldInstance, "add", new Object[]{i.next()})); + out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()})); } } } @@ -237,24 +241,26 @@ public static class SetDelegate extends DefaultPersistenceDelegate { @Override protected Expression instantiate(Object oldInstance, Encoder out) { - Set oldSet = (Set)oldInstance; + Set oldSet = (Set) oldInstance; HashSet newSet = new HashSet(oldSet); return new Expression(newSet, HashSet.class, "new", new Object[] {}); } + @Override protected boolean mutatesTo(Object oldInstance, Object newInstance) { return false; } + @Override protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) { - java.util.Collection oldO = (java.util.Collection)oldInstance; - java.util.Collection newO = (java.util.Collection)newInstance; + java.util.Collection oldO = (java.util.Collection) oldInstance; + java.util.Collection newO = (java.util.Collection) newInstance; if (newO.size() != 0) { - out.writeStatement(new Statement(oldInstance, "clear", new Object[]{})); + out.writeStatement(new Statement(oldInstance, "clear", new Object[] {})); } for (Iterator i = oldO.iterator(); i.hasNext();) { - out.writeStatement(new Statement(oldInstance, "add", new Object[]{i.next()})); + out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()})); } } @@ -263,24 +269,26 @@ public static class ListDelegate extends DefaultPersistenceDelegate { @Override protected Expression instantiate(Object oldInstance, Encoder out) { - List oldList = (List)oldInstance; + List oldList = (List) oldInstance; ArrayList newList = new ArrayList(oldList); return new Expression(newList, ArrayList.class, "new", new Object[] {}); } + @Override protected boolean mutatesTo(Object oldInstance, Object newInstance) { return false; } + @Override protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) { - java.util.Collection oldO = (java.util.Collection)oldInstance; - java.util.Collection newO = (java.util.Collection)newInstance; + java.util.Collection oldO = (java.util.Collection) oldInstance; + java.util.Collection newO = (java.util.Collection) newInstance; if (newO.size() != 0) { - out.writeStatement(new Statement(oldInstance, "clear", new Object[]{})); + out.writeStatement(new Statement(oldInstance, "clear", new Object[] {})); } for (Iterator i = oldO.iterator(); i.hasNext();) { - out.writeStatement(new Statement(oldInstance, "add", new Object[]{i.next()})); + out.writeStatement(new Statement(oldInstance, "add", new Object[] {i.next()})); } } @@ -297,12 +305,12 @@ // Serialize the plan to the default hdfs instance // Except for hadoop local mode execution where we should be // able to get the plan directly from the cache - if(!HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local")) { + if (!HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local")) { // use the default file system of the job FileSystem fs = planPath.getFileSystem(job); FSDataOutputStream out = fs.create(planPath); serializeMapRedWork(w, out); - + // Set up distributed cache DistributedCache.createSymlink(job); String uriWithLink = planPath.toUri().toString() + "#HIVE_PLAN" + jobID; @@ -310,7 +318,7 @@ // set replication of the plan file to a high number. we use the same // replication factor as used by the hadoop jobclient for job.xml etc. - short replication = (short)job.getInt("mapred.submit.replication", 10); + short replication = (short) job.getInt("mapred.submit.replication", 10); fs.setReplication(planPath, replication); } @@ -324,7 +332,7 @@ } public static String getHiveJobID(Configuration job) { - String planPath= HiveConf.getVar(job, HiveConf.ConfVars.PLAN); + String planPath = HiveConf.getVar(job, HiveConf.ConfVars.PLAN); if (planPath != null) { return (new Path(planPath)).getName(); } @@ -346,17 +354,15 @@ } } - public static ExprNodeDesc deserializeExpression( - String s, Configuration conf) { - byte [] bytes; + public static ExprNodeDesc deserializeExpression(String s, Configuration conf) { + byte[] bytes; try { bytes = s.getBytes("UTF-8"); } catch (UnsupportedEncodingException ex) { throw new RuntimeException("UTF-8 support required", ex); } ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - XMLDecoder decoder = new XMLDecoder( - bais, null, null, conf.getClassLoader()); + XMLDecoder decoder = new XMLDecoder(bais, null, null, conf.getClassLoader()); try { ExprNodeDesc expr = (ExprNodeDesc) decoder.readObject(); return expr; @@ -368,36 +374,29 @@ /** * Serialize a single Task. */ - public static void serializeTasks(Task t, - OutputStream out) { + public static void serializeTasks(Task t, OutputStream out) { XMLEncoder e = new XMLEncoder(out); // workaround for java 1.5 e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate()); e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate()); - e.setPersistenceDelegate(Operator.ProgressCounter.class, - new EnumDelegate()); + e.setPersistenceDelegate(Operator.ProgressCounter.class, new EnumDelegate()); e.writeObject(t); e.close(); } - public static class CollectionPersistenceDelegate extends DefaultPersistenceDelegate { + public static class CollectionPersistenceDelegate extends DefaultPersistenceDelegate { @Override protected Expression instantiate(Object oldInstance, Encoder out) { - return new Expression(oldInstance, - oldInstance.getClass(), - "new", - null); + return new Expression(oldInstance, oldInstance.getClass(), "new", null); } @Override - protected void initialize(Class type, Object oldInstance, Object newInstance, - Encoder out) { + protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) { Iterator ite = ((Collection) oldInstance).iterator(); while (ite.hasNext()) { - out.writeStatement(new Statement(oldInstance, "add", - new Object[] { ite.next() })); - } + out.writeStatement(new Statement(oldInstance, "add", new Object[] {ite.next()})); + } } } @@ -415,8 +414,7 @@ // workaround for java 1.5 e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate()); e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate()); - e.setPersistenceDelegate(Operator.ProgressCounter.class, - new EnumDelegate()); + e.setPersistenceDelegate(Operator.ProgressCounter.class, new EnumDelegate()); e.setPersistenceDelegate(org.datanucleus.store.types.sco.backed.Map.class, new MapDelegate()); e.setPersistenceDelegate(org.datanucleus.store.types.sco.backed.List.class, new ListDelegate()); @@ -428,8 +426,7 @@ /** * Deserialize the whole query plan. */ - public static QueryPlan deserializeQueryPlan(InputStream in, - Configuration conf) { + public static QueryPlan deserializeQueryPlan(InputStream in, Configuration conf) { XMLDecoder d = new XMLDecoder(in, null, null, conf.getClassLoader()); QueryPlan ret = (QueryPlan) d.readObject(); d.close(); @@ -437,9 +434,8 @@ } /** - * Serialize the mapredWork object to an output stream. DO NOT use this to - * write to standard output since it closes the output stream. - * DO USE mapredWork.toXML() instead. + * Serialize the mapredWork object to an output stream. DO NOT use this to write to standard + * output since it closes the output stream. DO USE mapredWork.toXML() instead. */ public static void serializeMapRedWork(MapredWork w, OutputStream out) { XMLEncoder e = new XMLEncoder(out); @@ -450,8 +446,7 @@ e.close(); } - public static MapredWork deserializeMapRedWork(InputStream in, - Configuration conf) { + public static MapredWork deserializeMapRedWork(InputStream in, Configuration conf) { XMLDecoder d = new XMLDecoder(in, null, null, conf.getClassLoader()); MapredWork ret = (MapredWork) d.readObject(); d.close(); @@ -459,6 +454,26 @@ } /** + * Serialize the mapredLocalWork object to an output stream. DO NOT use this to write to standard + * output since it closes the output stream. DO USE mapredWork.toXML() instead. + */ + public static void serializeMapRedLocalWork(MapredLocalWork w, OutputStream out) { + XMLEncoder e = new XMLEncoder(out); + // workaround for java 1.5 + e.setPersistenceDelegate(ExpressionTypes.class, new EnumDelegate()); + e.setPersistenceDelegate(GroupByDesc.Mode.class, new EnumDelegate()); + e.writeObject(w); + e.close(); + } + + public static MapredLocalWork deserializeMapRedLocalWork(InputStream in, Configuration conf) { + XMLDecoder d = new XMLDecoder(in, null, null, conf.getClassLoader()); + MapredLocalWork ret = (MapredLocalWork) d.readObject(); + d.close(); + return (ret); + } + + /** * Tuple. * * @param @@ -507,19 +522,17 @@ public static Random randGen = new Random(); /** - * Gets the task id if we are running as a Hadoop job. Gets a random number - * otherwise. + * Gets the task id if we are running as a Hadoop job. Gets a random number otherwise. */ public static String getTaskId(Configuration hconf) { String taskid = (hconf == null) ? null : hconf.get("mapred.task.id"); if ((taskid == null) || taskid.equals("")) { return ("" + Math.abs(randGen.nextInt())); } else { - /* extract the task and attempt id from the hadoop taskid. - in version 17 the leading component was 'task_'. thereafter - the leading component is 'attempt_'. in 17 - hadoop also - seems to have used _map_ and _reduce_ to denote map/reduce - task types + /* + * extract the task and attempt id from the hadoop taskid. in version 17 the leading component + * was 'task_'. thereafter the leading component is 'attempt_'. in 17 - hadoop also seems to + * have used _map_ and _reduce_ to denote map/reduce task types */ String ret = taskid.replaceAll(".*_[mr]_", "").replaceAll(".*_(map|reduce)_", ""); return (ret); @@ -587,29 +600,26 @@ } public static TableDesc getTableDesc(Table tbl) { - return (new TableDesc(tbl.getDeserializer().getClass(), tbl - .getInputFormatClass(), tbl.getOutputFormatClass(), tbl.getSchema())); + return (new TableDesc(tbl.getDeserializer().getClass(), tbl.getInputFormatClass(), tbl + .getOutputFormatClass(), tbl.getSchema())); } // column names and column types are all delimited by comma public static TableDesc getTableDesc(String cols, String colTypes) { return (new TableDesc(LazySimpleSerDe.class, SequenceFileInputFormat.class, HiveSequenceFileOutputFormat.class, Utilities.makeProperties( - org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "" - + Utilities.ctrlaCode, - org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS, cols, - org.apache.hadoop.hive.serde.Constants.LIST_COLUMN_TYPES, colTypes))); + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, + org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS, cols, + org.apache.hadoop.hive.serde.Constants.LIST_COLUMN_TYPES, colTypes))); } - public static PartitionDesc getPartitionDesc(Partition part) - throws HiveException { + public static PartitionDesc getPartitionDesc(Partition part) throws HiveException { return (new PartitionDesc(part)); } - public static void addMapWork(MapredWork mr, Table tbl, String alias, - Operator work) { - mr.addMapWork(tbl.getDataLocation().getPath(), alias, work, - new PartitionDesc(getTableDesc(tbl), null)); + public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator work) { + mr.addMapWork(tbl.getDataLocation().getPath(), alias, work, new PartitionDesc( + getTableDesc(tbl), null)); } private static String getOpTreeSkel_helper(Operator op, String indent) { @@ -641,8 +651,8 @@ return Character.isWhitespace((char) c); } - public static boolean contentsEqual(InputStream is1, InputStream is2, - boolean ignoreWhitespace) throws IOException { + public static boolean contentsEqual(InputStream is1, InputStream is2, boolean ignoreWhitespace) + throws IOException { try { if ((is1 == is2) || (is1 == null && is2 == null)) { return true; @@ -710,8 +720,7 @@ EOF, TERMINATED } - public static StreamStatus readColumn(DataInput in, OutputStream out) - throws IOException { + public static StreamStatus readColumn(DataInput in, OutputStream out) throws IOException { while (true) { int b; @@ -731,8 +740,8 @@ } /** - * Convert an output stream to a compressed output stream based on codecs and - * compression options specified in the Job Configuration. + * Convert an output stream to a compressed output stream based on codecs and compression options + * specified in the Job Configuration. * * @param jc * Job Configuration @@ -747,9 +756,8 @@ } /** - * Convert an output stream to a compressed output stream based on codecs - * codecs in the Job Configuration. Caller specifies directly whether file is - * compressed or not + * Convert an output stream to a compressed output stream based on codecs codecs in the Job + * Configuration. Caller specifies directly whether file is compressed or not * * @param jc * Job Configuration @@ -759,11 +767,11 @@ * whether the output stream needs to be compressed or not * @return compressed output stream */ - public static OutputStream createCompressedStream(JobConf jc, - OutputStream out, boolean isCompressed) throws IOException { + public static OutputStream createCompressedStream(JobConf jc, OutputStream out, + boolean isCompressed) throws IOException { if (isCompressed) { - Class codecClass = FileOutputFormat - .getOutputCompressorClass(jc, DefaultCodec.class); + Class codecClass = FileOutputFormat.getOutputCompressorClass(jc, + DefaultCodec.class); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc); return codec.createOutputStream(out); } else { @@ -772,8 +780,8 @@ } /** - * Based on compression option and configured output codec - get extension for - * output file. This is only required for text files - not sequencefiles + * Based on compression option and configured output codec - get extension for output file. This + * is only required for text files - not sequencefiles * * @param jc * Job Configuration @@ -785,8 +793,8 @@ if (!isCompressed) { return ""; } else { - Class codecClass = FileOutputFormat - .getOutputCompressorClass(jc, DefaultCodec.class); + Class codecClass = FileOutputFormat.getOutputCompressorClass(jc, + DefaultCodec.class); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc); return codec.getDefaultExtension(); } @@ -807,17 +815,15 @@ * Java Class for value * @return output stream over the created sequencefile */ - public static SequenceFile.Writer createSequenceWriter(JobConf jc, - FileSystem fs, Path file, Class keyClass, Class valClass) - throws IOException { + public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file, + Class keyClass, Class valClass) throws IOException { boolean isCompressed = FileOutputFormat.getCompressOutput(jc); return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed); } /** - * Create a sequencefile output stream based on job configuration Uses user - * supplied compression flag (rather than obtaining it from the Job - * Configuration). + * Create a sequencefile output stream based on job configuration Uses user supplied compression + * flag (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration @@ -831,26 +837,23 @@ * Java Class for value * @return output stream over the created sequencefile */ - public static SequenceFile.Writer createSequenceWriter(JobConf jc, - FileSystem fs, Path file, Class keyClass, Class valClass, - boolean isCompressed) throws IOException { + public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file, + Class keyClass, Class valClass, boolean isCompressed) throws IOException { CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; Class codecClass = null; if (isCompressed) { compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc); - codecClass = FileOutputFormat.getOutputCompressorClass(jc, - DefaultCodec.class); + codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc); } - return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass, - compressionType, codec)); + return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec)); } /** - * Create a RCFile output stream based on job configuration Uses user supplied - * compression flag (rather than obtaining it from the Job Configuration). + * Create a RCFile output stream based on job configuration Uses user supplied compression flag + * (rather than obtaining it from the Job Configuration). * * @param jc * Job configuration @@ -860,13 +863,12 @@ * Path to be created * @return output stream over the created rcfile */ - public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, - Path file, boolean isCompressed) throws IOException { + public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file, + boolean isCompressed) throws IOException { CompressionCodec codec = null; Class codecClass = null; if (isCompressed) { - codecClass = FileOutputFormat.getOutputCompressorClass(jc, - DefaultCodec.class); + codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc); } return new RCFile.Writer(fs, jc, file, null, codec); @@ -875,8 +877,7 @@ /** * Shamelessly cloned from GenericOptionsParser. */ - public static String realFile(String newFile, Configuration conf) - throws IOException { + public static String realFile(String newFile, Configuration conf) throws IOException { Path path = new Path(newFile); URI pathURI = path.toUri(); FileSystem fs; @@ -897,8 +898,7 @@ } String file = path.makeQualified(fs).toString(); // For compatibility with hadoop 0.17, change file:/a/b/c to file:///a/b/c - if (StringUtils.startsWith(file, "file:/") - && !StringUtils.startsWith(file, "file:///")) { + if (StringUtils.startsWith(file, "file:/") && !StringUtils.startsWith(file, "file:///")) { file = "file:///" + file.substring("file:/".length()); } return file; @@ -950,9 +950,8 @@ } /** - * Rename src to dst, or in the case dst already exists, move files in src to - * dst. If there is an existing file with the same name, the new file's name - * will be appended with "_1", "_2", etc. + * Rename src to dst, or in the case dst already exists, move files in src to dst. If there is an + * existing file with the same name, the new file's name will be appended with "_1", "_2", etc. * * @param fs * the FileSystem where src and dst are on. @@ -962,17 +961,15 @@ * the target directory * @throws IOException */ - public static void rename(FileSystem fs, Path src, Path dst) - throws IOException, HiveException { + public static void rename(FileSystem fs, Path src, Path dst) throws IOException, HiveException { if (!fs.rename(src, dst)) { throw new HiveException("Unable to move: " + src + " to: " + dst); } } /** - * Rename src to dst, or in the case dst already exists, move files in src to - * dst. If there is an existing file with the same name, the new file's name - * will be appended with "_1", "_2", etc. + * Rename src to dst, or in the case dst already exists, move files in src to dst. If there is an + * existing file with the same name, the new file's name will be appended with "_1", "_2", etc. * * @param fs * the FileSystem where src and dst are on. @@ -982,8 +979,8 @@ * the target directory * @throws IOException */ - public static void renameOrMoveFiles(FileSystem fs, Path src, Path dst) - throws IOException, HiveException { + public static void renameOrMoveFiles(FileSystem fs, Path src, Path dst) throws IOException, + HiveException { if (!fs.exists(dst)) { if (!fs.rename(src, dst)) { throw new HiveException("Unable to move: " + src + " to: " + dst); @@ -1010,18 +1007,18 @@ } /** - * The first group will contain the task id. The second group is the optional - * extension. The file name looks like: "0_0" or "0_0.gz". There may be a leading - * prefix (tmp_). Since getTaskId() can return an integer only - this should match - * a pure integer as well + * The first group will contain the task id. The second group is the optional extension. The file + * name looks like: "0_0" or "0_0.gz". There may be a leading prefix (tmp_). Since getTaskId() can + * return an integer only - this should match a pure integer as well */ private static Pattern fileNameTaskIdRegex = Pattern.compile("^.*?([0-9]+)(_[0-9])?(\\..*)?$"); /** - * Get the task id from the filename. - * It is assumed that the filename is derived from the output of getTaskId + * Get the task id from the filename. It is assumed that the filename is derived from the output + * of getTaskId * - * @param filename filename to extract taskid from + * @param filename + * filename to extract taskid from */ public static String getTaskIdFromFilename(String filename) { String taskId = filename; @@ -1032,8 +1029,8 @@ Matcher m = fileNameTaskIdRegex.matcher(taskId); if (!m.matches()) { - LOG.warn("Unable to get task id from file name: " + filename - + ". Using last component" + taskId + " as task id."); + LOG.warn("Unable to get task id from file name: " + filename + ". Using last component" + + taskId + " as task id."); } else { taskId = m.group(1); } @@ -1042,17 +1039,16 @@ } /** - * Replace the task id from the filename. - * It is assumed that the filename is derived from the output of getTaskId + * Replace the task id from the filename. It is assumed that the filename is derived from the + * output of getTaskId * - * @param filename filename to replace taskid - * "0_0" or "0_0.gz" by 33 to - * "33_0" or "33_0.gz" + * @param filename + * filename to replace taskid "0_0" or "0_0.gz" by 33 to "33_0" or "33_0.gz" */ public static String replaceTaskIdFromFilename(String filename, int bucketNum) { String taskId = getTaskIdFromFilename(filename); String newTaskId = replaceTaskId(taskId, bucketNum); - String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); + String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); return (ret); } @@ -1062,21 +1058,22 @@ int taskIdLen = taskId.length(); StringBuffer s = new StringBuffer(); for (int i = 0; i < taskIdLen - bucketNumLen; i++) { - s.append("0"); + s.append("0"); } return s.toString() + strBucketNum; } /** - * Replace the oldTaskId appearing in the filename by the newTaskId. - * The string oldTaskId could appear multiple times, we should only replace the last one. + * Replace the oldTaskId appearing in the filename by the newTaskId. The string oldTaskId could + * appear multiple times, we should only replace the last one. + * * @param filename * @param oldTaskId * @param newTaskId * @return */ - private static String replaceTaskIdFromFilename(String filename, - String oldTaskId, String newTaskId) { + private static String replaceTaskIdFromFilename(String filename, String oldTaskId, + String newTaskId) { String[] spl = filename.split(oldTaskId); @@ -1085,27 +1082,31 @@ } StringBuffer snew = new StringBuffer(); - for (int idx = 0; idx < spl.length-1; idx++) { + for (int idx = 0; idx < spl.length - 1; idx++) { if (idx > 0) { snew.append(oldTaskId); } snew.append(spl[idx]); } snew.append(newTaskId); - snew.append(spl[spl.length-1]); + snew.append(spl[spl.length - 1]); return snew.toString(); } /** * Get all file status from a root path and recursively go deep into certain levels. - * @param path the root path - * @param level the depth of directory should explore - * @param fs the file system + * + * @param path + * the root path + * @param level + * the depth of directory should explore + * @param fs + * the file system * @return array of FileStatus * @throws IOException */ - public static FileStatus[] getFileStatusRecurse(Path path, int level, - FileSystem fs) throws IOException { + public static FileStatus[] getFileStatusRecurse(Path path, int level, FileSystem fs) + throws IOException { // construct a path pattern (e.g., /*/*) to find all dynamically generated paths StringBuilder sb = new StringBuilder(path.toUri().getPath()); @@ -1117,8 +1118,8 @@ } /** - * Remove all temporary files and duplicate (double-committed) files from a - * given directory. + * Remove all temporary files and duplicate (double-committed) files from a given directory. + * * @return a list of path names corresponding to should-be-created empty buckets. */ public static void removeTempOrDuplicateFiles(FileSystem fs, Path path) throws IOException { @@ -1126,12 +1127,12 @@ } /** - * Remove all temporary files and duplicate (double-committed) files from a - * given directory. + * Remove all temporary files and duplicate (double-committed) files from a given directory. + * * @return a list of path names corresponding to should-be-created empty buckets. */ - public static ArrayList removeTempOrDuplicateFiles(FileSystem fs, Path path, DynamicPartitionCtx dpCtx) - throws IOException { + public static ArrayList removeTempOrDuplicateFiles(FileSystem fs, Path path, + DynamicPartitionCtx dpCtx) throws IOException { if (path == null) { return null; } @@ -1142,7 +1143,8 @@ HashMap taskIDToFile = null; for (int i = 0; i < parts.length; ++i) { - assert parts[i].isDir(): "dynamic partition " + parts[i].getPath() + " is not a direcgtory"; + assert parts[i].isDir() : "dynamic partition " + parts[i].getPath() + + " is not a direcgtory"; FileStatus[] items = fs.listStatus(parts[i].getPath()); // remove empty directory since DP insert should not generate empty partitions. @@ -1162,7 +1164,7 @@ // get the missing buckets and generate empty buckets String taskID1 = taskIDToFile.keySet().iterator().next(); Path bucketPath = taskIDToFile.values().iterator().next().getPath(); - for (int j = 0; j < dpCtx.getNumBuckets(); ++j ) { + for (int j = 0; j < dpCtx.getNumBuckets(); ++j) { String taskID2 = replaceTaskId(taskID1, j); if (!taskIDToFile.containsKey(taskID2)) { // create empty bucket, file name should be derived from taskID2 @@ -1177,11 +1179,10 @@ removeTempOrDuplicateFiles(items, fs); } return result; - } + } - public static HashMap removeTempOrDuplicateFiles( - FileStatus[] items, FileSystem fs) - throws IOException { + public static HashMap removeTempOrDuplicateFiles(FileStatus[] items, + FileSystem fs) throws IOException { if (items == null || fs == null) { return null; @@ -1214,12 +1215,12 @@ long len1 = toDelete.getLen(); long len2 = taskIdToFile.get(taskId).getLen(); if (!fs.delete(toDelete.getPath(), true)) { - throw new IOException("Unable to delete duplicate file: " - + toDelete.getPath() + ". Existing file: " + taskIdToFile.get(taskId).getPath()); + throw new IOException("Unable to delete duplicate file: " + toDelete.getPath() + + ". Existing file: " + taskIdToFile.get(taskId).getPath()); } else { LOG.warn("Duplicate taskid file removed: " + toDelete.getPath() + " with length " - + len1 + ". Existing file: " + taskIdToFile.get(taskId).getPath() - + " with length " + len2); + + len1 + ". Existing file: " + taskIdToFile.get(taskId).getPath() + " with length " + + len2); } } } @@ -1237,8 +1238,7 @@ * @param newPaths * Array of classpath elements */ - public static ClassLoader addToClassPath(ClassLoader cloader, - String[] newPaths) throws Exception { + public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception { URLClassLoader loader = (URLClassLoader) cloader; List curPath = Arrays.asList(loader.getURLs()); ArrayList newPath = new ArrayList(); @@ -1270,8 +1270,7 @@ * @param pathsToRemove * Array of classpath elements */ - public static void removeFromClassPath(String[] pathsToRemove) - throws Exception { + public static void removeFromClassPath(String[] pathsToRemove) throws Exception { Thread curThread = Thread.currentThread(); URLClassLoader loader = (URLClassLoader) curThread.getContextClassLoader(); Set newPath = new HashSet(Arrays.asList(loader.getURLs())); @@ -1307,8 +1306,7 @@ return names; } - public static List getColumnNamesFromFieldSchema( - List partCols) { + public static List getColumnNamesFromFieldSchema(List partCols) { List names = new ArrayList(); for (FieldSchema o : partCols) { names.add(o.getName()); @@ -1344,8 +1342,8 @@ return names; } - public static void validateColumnNames(List colNames, - List checkCols) throws SemanticException { + public static void validateColumnNames(List colNames, List checkCols) + throws SemanticException { Iterator checkColsIter = checkCols.iterator(); while (checkColsIter.hasNext()) { String toCheck = checkColsIter.next(); @@ -1365,16 +1363,15 @@ } /** - * Gets the default notification interval to send progress updates to the - * tracker. Useful for operators that may not output data for a while. + * Gets the default notification interval to send progress updates to the tracker. Useful for + * operators that may not output data for a while. * * @param hconf * @return the interval in milliseconds */ public static int getDefaultNotificationInterval(Configuration hconf) { int notificationInterval; - Integer expInterval = Integer.decode(hconf - .get("mapred.tasktracker.expiry.interval")); + Integer expInterval = Integer.decode(hconf.get("mapred.tasktracker.expiry.interval")); if (expInterval != null) { notificationInterval = expInterval.intValue() / 2; @@ -1386,12 +1383,14 @@ } /** - * Copies the storage handler properties configured for a table descriptor - * to a runtime job configuration. + * Copies the storage handler properties configured for a table descriptor to a runtime job + * configuration. * - * @param tbl table descriptor from which to read + * @param tbl + * table descriptor from which to read * - * @param job configuration which receives configured properties + * @param job + * configuration which receives configured properties */ public static void copyTableJobPropertiesToConf(TableDesc tbl, JobConf job) { Map jobProperties = tbl.getJobProperties(); @@ -1406,14 +1405,17 @@ /** * Calculate the total size of input files. * - * @param job the hadoop job conf. - * @param work map reduce job plan - * @param filter filter to apply to the input paths before calculating size + * @param job + * the hadoop job conf. + * @param work + * map reduce job plan + * @param filter + * filter to apply to the input paths before calculating size * @return the summary of all the input paths. * @throws IOException */ - public static ContentSummary getInputSummary - (Context ctx, MapredWork work, PathFilter filter) throws IOException { + public static ContentSummary getInputSummary(Context ctx, MapredWork work, PathFilter filter) + throws IOException { long[] summary = {0, 0, 0}; @@ -1422,7 +1424,7 @@ try { Path p = new Path(path); - if(filter != null && !filter.accept(p)) { + if (filter != null && !filter.accept(p)) { continue; } @@ -1459,19 +1461,18 @@ return true; } - public static List getMRTasks (List> tasks) { - List mrTasks = new ArrayList (); - if(tasks != null) { + public static List getMRTasks(List> tasks) { + List mrTasks = new ArrayList(); + if (tasks != null) { getMRTasks(tasks, mrTasks); } return mrTasks; } - private static void getMRTasks (List> tasks, - List mrTasks) { + private static void getMRTasks(List> tasks, List mrTasks) { for (Task task : tasks) { - if (task instanceof ExecDriver && !mrTasks.contains((ExecDriver)task)) { - mrTasks.add((ExecDriver)task); + if (task instanceof ExecDriver && !mrTasks.contains((ExecDriver) task)) { + mrTasks.add((ExecDriver) task); } if (task.getDependentTasks() != null) { @@ -1485,45 +1486,43 @@ } /** - * Construct a list of full partition spec from Dynamic Partition Context and - * the directory names corresponding to these dynamic partitions. + * Construct a list of full partition spec from Dynamic Partition Context and the directory names + * corresponding to these dynamic partitions. */ public static List> getFullDPSpecs(Configuration conf, - DynamicPartitionCtx dpCtx) - throws HiveException { + DynamicPartitionCtx dpCtx) throws HiveException { try { Path loadPath = new Path(dpCtx.getRootPath()); FileSystem fs = loadPath.getFileSystem(conf); - int numDPCols = dpCtx.getNumDPCols(); - FileStatus[] status = Utilities.getFileStatusRecurse(loadPath, numDPCols, fs); + int numDPCols = dpCtx.getNumDPCols(); + FileStatus[] status = Utilities.getFileStatusRecurse(loadPath, numDPCols, fs); - if (status.length == 0) { - LOG.warn("No partition is genereated by dynamic partitioning"); - return null; - } + if (status.length == 0) { + LOG.warn("No partition is genereated by dynamic partitioning"); + return null; + } - // partial partition specification - Map partSpec = dpCtx.getPartSpec(); + // partial partition specification + Map partSpec = dpCtx.getPartSpec(); - // list of full partition specification - List> fullPartSpecs = - new ArrayList>(); + // list of full partition specification + List> fullPartSpecs = new ArrayList>(); - // for each dynamically created DP directory, construct a full partition spec - // and load the partition based on that - for (int i= 0; i < status.length; ++i) { - // get the dynamically created directory - Path partPath = status[i].getPath(); - assert fs.getFileStatus(partPath).isDir(): - "partitions " + partPath + " is not a directory !"; + // for each dynamically created DP directory, construct a full partition spec + // and load the partition based on that + for (int i = 0; i < status.length; ++i) { + // get the dynamically created directory + Path partPath = status[i].getPath(); + assert fs.getFileStatus(partPath).isDir() : "partitions " + partPath + + " is not a directory !"; - // generate a full partition specification - LinkedHashMap fullPartSpec = new LinkedHashMap(partSpec); - Warehouse.makeSpecFromName(fullPartSpec, partPath); - fullPartSpecs.add(fullPartSpec); - } - return fullPartSpecs; + // generate a full partition specification + LinkedHashMap fullPartSpec = new LinkedHashMap(partSpec); + Warehouse.makeSpecFromName(fullPartSpec, partPath); + fullPartSpecs.add(fullPartSpec); + } + return fullPartSpecs; } catch (IOException e) { throw new HiveException(e); } @@ -1551,9 +1550,7 @@ columnNames.append(colInfo.getInternalName()); } String columnNamesString = columnNames.toString(); - jobConf.set( - Constants.LIST_COLUMNS, - columnNamesString); + jobConf.set(Constants.LIST_COLUMNS, columnNamesString); } public static void validatePartSpec(Table tbl, Map partSpec) @@ -1561,13 +1558,37 @@ List parts = tbl.getPartitionKeys(); Set partCols = new HashSet(parts.size()); - for (FieldSchema col: parts) { + for (FieldSchema col : parts) { partCols.add(col.getName()); } - for (String col: partSpec.keySet()) { + for (String col : partSpec.keySet()) { if (!partCols.contains(col)) { throw new SemanticException(ErrorMsg.NONEXISTPARTCOL.getMsg(col)); } } } + + public static String suffix = ".hashtable"; + + public static String generatePath(String baseURI, Byte tag, String bigBucketFileName) { + String path = new String(baseURI + Path.SEPARATOR + "-" + tag + "-" + bigBucketFileName + + suffix); + return path; + } + + public static String generateFileName(Byte tag, String bigBucketFileName) { + String fileName = new String("-" + tag + "-" + bigBucketFileName + suffix); + return fileName; + } + + public static String generateTmpURI(String baseURI, String id) { + String tmpFileURI = new String(baseURI + Path.SEPARATOR + "HashTable-" + id); + return tmpFileURI; + } + + public static String now() { + Calendar cal = Calendar.getInstance(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + return sdf.format(cal.getTime()); + } } Index: src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/JDBMSinkOperator.java (working copy) @@ -1,483 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectKey; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectValue; -import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.JDBMSinkDesc; -import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; -import org.apache.hadoop.hive.ql.util.JoinUtil; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.util.ReflectionUtils; - - -public class JDBMSinkOperator extends TerminalOperator -implements Serializable { - private static final long serialVersionUID = 1L; - private static final Log LOG = LogFactory.getLog(JDBMSinkOperator.class - .getName()); - - //from abstract map join operator - /** - * The expressions for join inputs's join keys. - */ - protected transient Map> joinKeys; - /** - * The ObjectInspectors for the join inputs's join keys. - */ - protected transient Map> joinKeysObjectInspectors; - /** - * The standard ObjectInspectors for the join inputs's join keys. - */ - protected transient Map> joinKeysStandardObjectInspectors; - - protected transient int posBigTableTag = -1; // one of the tables that is not in memory - protected transient int posBigTableAlias = -1; // one of the tables that is not in memory - transient int mapJoinRowsKey; // rows for a given key - - protected transient RowContainer> emptyList = null; - - transient int numMapRowsRead; - protected transient int totalSz; // total size of the composite object - transient boolean firstRow; - private boolean smallTablesOnly; - /** - * The filters for join - */ - protected transient Map> joinFilters; - - protected transient int numAliases; // number of aliases - /** - * The expressions for join outputs. - */ - protected transient Map> joinValues; - /** - * The ObjectInspectors for the join inputs. - */ - protected transient Map> joinValuesObjectInspectors; - /** - * The ObjectInspectors for join filters. - */ - protected transient Map> joinFilterObjectInspectors; - /** - * The standard ObjectInspectors for the join inputs. - */ - protected transient Map> joinValuesStandardObjectInspectors; - - protected transient - Map> rowContainerStandardObjectInspectors; - - protected transient Byte[] order; // order in which the results should - Configuration hconf; - protected transient Byte alias; - protected transient Map spillTableDesc; // spill tables are - - protected transient Map> mapJoinTables; - protected transient boolean noOuterJoin; - - public static class JDBMSinkObjectCtx { - ObjectInspector standardOI; - SerDe serde; - TableDesc tblDesc; - Configuration conf; - - /** - * @param standardOI - * @param serde - */ - public JDBMSinkObjectCtx(ObjectInspector standardOI, SerDe serde, - TableDesc tblDesc, Configuration conf) { - this.standardOI = standardOI; - this.serde = serde; - this.tblDesc = tblDesc; - this.conf = conf; - } - - /** - * @return the standardOI - */ - public ObjectInspector getStandardOI() { - return standardOI; - } - - /** - * @return the serde - */ - public SerDe getSerDe() { - return serde; - } - - public TableDesc getTblDesc() { - return tblDesc; - } - - public Configuration getConf() { - return conf; - } - - } - - private static final transient String[] FATAL_ERR_MSG = { - null, // counter value 0 means no error - "Mapside join size exceeds hive.mapjoin.maxsize. " - + "Please increase that or remove the mapjoin hint." - }; - transient int metadataKeyTag; - transient int[] metadataValueTag; - transient int maxMapJoinSize; - - - public JDBMSinkOperator(){ - //super(); - } - - public JDBMSinkOperator(MapJoinOperator mjop){ - this.conf = new JDBMSinkDesc(mjop.getConf()); - } - - - @Override - protected void initializeOp(Configuration hconf) throws HiveException { - - maxMapJoinSize = HiveConf.getIntVar(hconf, - HiveConf.ConfVars.HIVEMAXMAPJOINSIZE); - - numMapRowsRead = 0; - firstRow = true; - - //for small tables only; so get the big table position first - posBigTableTag = conf.getPosBigTable(); - - order = conf.getTagOrder(); - - posBigTableAlias=order[posBigTableTag]; - - //initialize some variables, which used to be initialized in CommonJoinOperator - numAliases = conf.getExprs().size(); - this.hconf = hconf; - totalSz = 0; - - noOuterJoin = conf.isNoOuterJoin(); - - //process join keys - joinKeys = new HashMap>(); - JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(),order,posBigTableAlias); - joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys,inputObjInspectors,posBigTableAlias); - joinKeysStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(joinKeysObjectInspectors,posBigTableAlias); - - //process join values - joinValues = new HashMap>(); - JoinUtil.populateJoinKeyValue(joinValues, conf.getExprs(),order,posBigTableAlias); - joinValuesObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinValues,inputObjInspectors,posBigTableAlias); - joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(joinValuesObjectInspectors,posBigTableAlias); - - //process join filters - joinFilters = new HashMap>(); - JoinUtil.populateJoinKeyValue(joinFilters, conf.getFilters(),order,posBigTableAlias); - joinFilterObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinValues,inputObjInspectors,posBigTableAlias); - - - - - if (noOuterJoin) { - rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors; - } else { - Map> rowContainerObjectInspectors = - new HashMap>(); - for (Byte alias : order) { - if(alias == posBigTableAlias){ - continue; - } - ArrayList rcOIs = new ArrayList(); - rcOIs.addAll(joinValuesObjectInspectors.get(alias)); - // for each alias, add object inspector for boolean as the last element - rcOIs.add( - PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); - rowContainerObjectInspectors.put(alias, rcOIs); - } - rowContainerStandardObjectInspectors = - getStandardObjectInspectors(rowContainerObjectInspectors); - } - - metadataValueTag = new int[numAliases]; - for (int pos = 0; pos < numAliases; pos++) { - metadataValueTag[pos] = -1; - } - - mapJoinTables = new HashMap>(); - - // initialize the hash tables for other tables - for (Byte pos:order) { - if (pos == posBigTableTag) { - continue; - } - - int cacheSize = HiveConf.getIntVar(hconf, - HiveConf.ConfVars.HIVEMAPJOINCACHEROWS); - HashMapWrapper hashTable = new HashMapWrapper( - cacheSize); - - mapJoinTables.put(pos, hashTable); - } - } - - - - protected static HashMap> getStandardObjectInspectors( - Map> aliasToObjectInspectors) { - HashMap> result = new HashMap>(); - for (Entry> oiEntry : aliasToObjectInspectors - .entrySet()) { - Byte alias = oiEntry.getKey(); - List oiList = oiEntry.getValue(); - ArrayList fieldOIList = new ArrayList( - oiList.size()); - for (int i = 0; i < oiList.size(); i++) { - fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList - .get(i), ObjectInspectorCopyOption.WRITABLE)); - } - result.put(alias, fieldOIList); - } - return result; - - } - - /* - * This operator only process small tables - * Read the key/value pairs - * Load them into hashtable - */ - @Override - public void processOp(Object row, int tag) throws HiveException{ - //let the mapJoinOp process these small tables - try{ - alias = order[tag]; - //alias = (byte)tag; - - // compute keys and values as StandardObjects - ArrayList key = JoinUtil.computeKeys(row, joinKeys.get(alias), - joinKeysObjectInspectors.get(alias)); - - ArrayList value = JoinUtil.computeValues(row, joinValues.get(alias), - joinValuesObjectInspectors.get(alias),joinFilters.get(alias), - joinFilterObjectInspectors.get(alias), noOuterJoin); - - - if (firstRow) { - metadataKeyTag = -1; - - TableDesc keyTableDesc = conf.getKeyTblDesc(); - SerDe keySerializer = (SerDe) ReflectionUtils.newInstance( - keyTableDesc.getDeserializerClass(), null); - keySerializer.initialize(null, keyTableDesc.getProperties()); - - MapJoinMetaData.clear(); - MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), - new JDBMSinkObjectCtx( - ObjectInspectorUtils - .getStandardObjectInspector(keySerializer - .getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), keySerializer, - keyTableDesc, hconf)); - - firstRow = false; - } - - numMapRowsRead++; - - if ((numMapRowsRead > maxMapJoinSize)&& (counterNameToEnum != null)) { - // update counter - LOG - .warn("Too many rows in map join tables. Fatal error counter will be incremented!!"); - incrCounter(fatalErrorCntr, 1); - fatalError = true; - return; - } - - HashMapWrapper hashTable = mapJoinTables.get((byte) tag); - MapJoinObjectKey keyMap = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue o = hashTable.get(keyMap); - RowContainer res = null; - - boolean needNewKey = true; - if (o == null) { - int bucketSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE); - res = JoinUtil.getRowContainer(hconf, - rowContainerStandardObjectInspectors.get((byte)tag), - order[tag], bucketSize,spillTableDesc,conf,noOuterJoin); - - res.add(value); - } else { - res = o.getObj(); - res.add(value); - - if (hashTable.cacheSize() > 0) { - o.setObj(res); - needNewKey = false; - } - } - - if (metadataValueTag[tag] == -1) { - metadataValueTag[tag] = order[tag]; - - TableDesc valueTableDesc = conf.getValueTblDescs().get(tag); - SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc - .getDeserializerClass(), null); - valueSerDe.initialize(null, valueTableDesc.getProperties()); - - MapJoinMetaData.put(Integer.valueOf(metadataValueTag[tag]), - new JDBMSinkObjectCtx(ObjectInspectorUtils - .getStandardObjectInspector(valueSerDe.getObjectInspector(), - ObjectInspectorCopyOption.WRITABLE), valueSerDe, - valueTableDesc, hconf)); - } - - // Construct externalizable objects for key and value - if (needNewKey) { - MapJoinObjectKey keyObj = new MapJoinObjectKey(metadataKeyTag, key); - MapJoinObjectValue valueObj = new MapJoinObjectValue( - metadataValueTag[tag], res); - - //valueObj.setConf(hconf); - valueObj.setConf(hconf); - // This may potentially increase the size of the hashmap on the mapper - if (res.size() > mapJoinRowsKey) { - if (res.size() % 100 == 0) { - LOG.warn("Number of values for a given key " + keyObj + " are " - + res.size()); - LOG.warn("used memory " + Runtime.getRuntime().totalMemory()); - } - } - hashTable.put(keyObj, valueObj); - } - }catch (SerDeException e) { - e.printStackTrace(); - throw new HiveException(e); - } - - } - - - @Override - /* - * Flush the hashtable into jdbm file - * Load this jdbm file into HDFS only - */ - public void closeOp(boolean abort) throws HiveException{ - try{ - if(mapJoinTables != null) { - //get tmp file URI - String tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); - LOG.info("Get TMP URI: "+tmpURI); - - for (Map.Entry> hashTables : mapJoinTables.entrySet()) { - //get the key and value - Byte tag = hashTables.getKey(); - HashMapWrapper hashTable = hashTables.getValue(); - - //get the jdbm file and path - String jdbmFile = hashTable.flushMemoryCacheToPersistent(); - Path localPath = new Path(jdbmFile); - - //get current input file name - String bigBucketFileName = this.getExecContext().getCurrentBigBucketFile(); - if(bigBucketFileName == null ||bigBucketFileName.length()==0) { - bigBucketFileName="-"; - } - //get the tmp URI path; it will be a hdfs path if not local mode - Path tmpURIPath = new Path(tmpURI+Path.SEPARATOR+"-"+tag+"-"+bigBucketFileName+".jdbm"); - - //upload jdbm file to this HDFS - FileSystem fs = tmpURIPath.getFileSystem(this.getExecContext().getJc()); - fs.copyFromLocalFile(localPath, tmpURIPath); - LOG.info("Upload 1 JDBM File to: "+tmpURIPath); - //remove the original jdbm tmp file - hashTable.close(); - } - } - - super.closeOp(abort); - }catch(IOException e){ - LOG.error("Copy local file to HDFS error"); - e.printStackTrace(); - } - } - - /** - * Implements the getName function for the Node Interface. - * - * @return the name of the operator - */ - @Override - public String getName() { - return "JDBMSINK"; - } - - @Override - public int getType() { - return OperatorType.JDBMSINK; - } - - private void getPersistentFilePath(Map paths) throws HiveException{ - Map jdbmFilePaths = paths; - try{ - if(mapJoinTables != null) { - for (Map.Entry> hashTables : mapJoinTables.entrySet()) { - //hashTable.close(); - - Byte key = hashTables.getKey(); - HashMapWrapper hashTable = hashTables.getValue(); - - //get the jdbm file and path - String jdbmFile = hashTable.flushMemoryCacheToPersistent(); - Path localPath = new Path(jdbmFile); - - //insert into map - jdbmFilePaths.put(key, localPath); - } - } - }catch (Exception e){ - LOG.fatal("Get local JDBM file error"); - e.printStackTrace(); - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (working copy) @@ -30,12 +30,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.persistence.AbstractRowContainer; import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.util.JoinUtil; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -143,7 +143,7 @@ // input is too large // to fit in memory - HashMap>> storage; // map b/w table alias + HashMap>> storage; // map b/w table alias // to RowContainer int joinEmitInterval = -1; int joinCacheSize = 0; @@ -154,7 +154,7 @@ protected transient int countAfterReport; protected transient int heartbeatInterval; - private static final int NOTSKIPBIGTABLE = -1; + protected static final int NOTSKIPBIGTABLE = -1; public CommonJoinOperator() { } @@ -228,7 +228,7 @@ totalSz = 0; // Map that contains the rows for each alias - storage = new HashMap>>(); + storage = new HashMap>>(); numAliases = conf.getExprs().size(); @@ -340,7 +340,7 @@ public void startGroup() throws HiveException { LOG.trace("Join: Starting new group"); newGroupStarted = true; - for (RowContainer> alw : storage.values()) { + for (AbstractRowContainer> alw : storage.values()) { alw.clear(); } } @@ -664,7 +664,7 @@ if (aliasNum < numAliases) { // search for match in the rhs table - RowContainer> aliasRes = storage.get(order[aliasNum]); + AbstractRowContainer> aliasRes = storage.get(order[aliasNum]); for (ArrayList newObj = aliasRes.first(); newObj != null; newObj = aliasRes .next()) { @@ -735,7 +735,7 @@ return; } - RowContainer> alias = storage.get(order[aliasNum]); + AbstractRowContainer> alias = storage.get(order[aliasNum]); for (ArrayList row = alias.first(); row != null; row = alias.next()) { intObj.pushObj(row); genUniqueJoinObject(aliasNum + 1, intObj); @@ -755,7 +755,7 @@ boolean hasNulls = false; // Will be true if there are null entries for (int i = 0; i < numAliases; i++) { Byte alias = order[i]; - RowContainer> alw = storage.get(alias); + AbstractRowContainer> alw = storage.get(alias); if (alw.size() == 0) { alw.add((ArrayList) dummyObj[i]); hasNulls = true; @@ -776,7 +776,7 @@ // does any result need to be emitted for (int i = 0; i < numAliases; i++) { Byte alias = order[i]; - RowContainer> alw = storage.get(alias); + AbstractRowContainer> alw = storage.get(alias); if (alw.size() == 0) { if (noOuterJoin) { LOG.trace("No data for alias=" + i); @@ -831,7 +831,7 @@ @Override public void closeOp(boolean abort) throws HiveException { LOG.trace("Join Op close"); - for (RowContainer> alw : storage.values()) { + for (AbstractRowContainer> alw : storage.values()) { if (alw != null) { alw.clear(); // clean up the temp files } Index: src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/exec/MapJoinMetaData.java (working copy) @@ -17,21 +17,23 @@ */ package org.apache.hadoop.hive.ql.exec; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.JDBMSinkOperator.JDBMSinkObjectCtx; +import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator.HashTableSinkObjectCtx; public class MapJoinMetaData { - static transient Map mapMetadata = new HashMap(); + static transient Map mapMetadata = new HashMap(); + static ArrayList list = new ArrayList(); public MapJoinMetaData(){ } - public static void put(Integer key, JDBMSinkObjectCtx value){ + public static void put(Integer key, HashTableSinkObjectCtx value){ mapMetadata.put(key, value); } - public static JDBMSinkObjectCtx get(Integer key){ + public static HashTableSinkObjectCtx get(Integer key){ return mapMetadata.get(key); } @@ -39,4 +41,9 @@ mapMetadata.clear(); } + public static ArrayList getList(){ + list.clear(); + return list; + } + } Index: src/java/org/apache/hadoop/hive/ql/plan/JDBMSinkDesc.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/plan/JDBMSinkDesc.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/plan/JDBMSinkDesc.java (working copy) @@ -1,355 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Map.Entry; - -/** - * Map Join operator Descriptor implementation. - * - */ -@Explain(displayName = "JDBM Sink Operator") -public class JDBMSinkDesc extends JoinDesc implements Serializable { - private static final long serialVersionUID = 1L; - - - // used to handle skew join - private boolean handleSkewJoin = false; - private int skewKeyDefinition = -1; - private Map bigKeysDirMap; - private Map> smallKeysDirMap; - private Map skewKeysValuesTables; - - // alias to key mapping - private Map> exprs; - - // alias to filter mapping - private Map> filters; - - // used for create joinOutputObjectInspector - protected List outputColumnNames; - - // key:column output name, value:tag - private transient Map reversedExprs; - - // No outer join involved - protected boolean noOuterJoin; - - protected JoinCondDesc[] conds; - - protected Byte[] tagOrder; - private TableDesc keyTableDesc; - - - private Map> keys; - private TableDesc keyTblDesc; - private List valueTblDescs; - - private int posBigTable; - - private Map> retainList; - - private transient String bigTableAlias; - - private LinkedHashMap>> aliasBucketFileNameMapping; - private LinkedHashMap bucketFileNameMapping; - - public JDBMSinkDesc() { - bucketFileNameMapping = new LinkedHashMap(); - } - - public JDBMSinkDesc(MapJoinDesc clone) { - this.bigKeysDirMap = clone.getBigKeysDirMap(); - this.conds = clone.getConds(); - this.exprs= clone.getExprs(); - this.handleSkewJoin = clone.getHandleSkewJoin(); - this.keyTableDesc = clone.getKeyTableDesc(); - this.noOuterJoin = clone.getNoOuterJoin(); - this.outputColumnNames = clone.getOutputColumnNames(); - this.reversedExprs = clone.getReversedExprs(); - this.skewKeyDefinition = clone.getSkewKeyDefinition(); - this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); - this.smallKeysDirMap = clone.getSmallKeysDirMap(); - this.tagOrder = clone.getTagOrder(); - this.filters = clone.getFilters(); - - this.keys = clone.getKeys(); - this.keyTblDesc = clone.getKeyTblDesc(); - this.valueTblDescs = clone.getValueTblDescs(); - this.posBigTable = clone.getPosBigTable(); - this.retainList = clone.getRetainList(); - this.bigTableAlias = clone.getBigTableAlias(); - this.aliasBucketFileNameMapping = clone.getAliasBucketFileNameMapping(); - this.bucketFileNameMapping = clone.getBucketFileNameMapping(); - } - - - private void initRetainExprList() { - retainList = new HashMap>(); - Set>> set = exprs.entrySet(); - Iterator>> setIter = set.iterator(); - while (setIter.hasNext()) { - Entry> current = setIter.next(); - List list = new ArrayList(); - for (int i = 0; i < current.getValue().size(); i++) { - list.add(i); - } - retainList.put(current.getKey(), list); - } - } - - public boolean isHandleSkewJoin() { - return handleSkewJoin; - } - - @Override - public void setHandleSkewJoin(boolean handleSkewJoin) { - this.handleSkewJoin = handleSkewJoin; - } - - @Override - public int getSkewKeyDefinition() { - return skewKeyDefinition; - } - - @Override - public void setSkewKeyDefinition(int skewKeyDefinition) { - this.skewKeyDefinition = skewKeyDefinition; - } - - @Override - public Map getBigKeysDirMap() { - return bigKeysDirMap; - } - - @Override - public void setBigKeysDirMap(Map bigKeysDirMap) { - this.bigKeysDirMap = bigKeysDirMap; - } - - @Override - public Map> getSmallKeysDirMap() { - return smallKeysDirMap; - } - - @Override - public void setSmallKeysDirMap(Map> smallKeysDirMap) { - this.smallKeysDirMap = smallKeysDirMap; - } - - @Override - public Map getSkewKeysValuesTables() { - return skewKeysValuesTables; - } - - @Override - public void setSkewKeysValuesTables(Map skewKeysValuesTables) { - this.skewKeysValuesTables = skewKeysValuesTables; - } - - @Override - public Map> getExprs() { - return exprs; - } - - @Override - public void setExprs(Map> exprs) { - this.exprs = exprs; - } - - @Override - public Map> getFilters() { - return filters; - } - - @Override - public void setFilters(Map> filters) { - this.filters = filters; - } - - @Override - public List getOutputColumnNames() { - return outputColumnNames; - } - - @Override - public void setOutputColumnNames(List outputColumnNames) { - this.outputColumnNames = outputColumnNames; - } - - @Override - public Map getReversedExprs() { - return reversedExprs; - } - - @Override - public void setReversedExprs(Map reversedExprs) { - this.reversedExprs = reversedExprs; - } - - @Override - public boolean isNoOuterJoin() { - return noOuterJoin; - } - - @Override - public void setNoOuterJoin(boolean noOuterJoin) { - this.noOuterJoin = noOuterJoin; - } - - @Override - public JoinCondDesc[] getConds() { - return conds; - } - - @Override - public void setConds(JoinCondDesc[] conds) { - this.conds = conds; - } - - @Override - public Byte[] getTagOrder() { - return tagOrder; - } - - @Override - public void setTagOrder(Byte[] tagOrder) { - this.tagOrder = tagOrder; - } - - @Override - public TableDesc getKeyTableDesc() { - return keyTableDesc; - } - - @Override - public void setKeyTableDesc(TableDesc keyTableDesc) { - this.keyTableDesc = keyTableDesc; - } - - - public Map> getRetainList() { - return retainList; - } - - public void setRetainList(Map> retainList) { - this.retainList = retainList; - } - - /** - * @return the keys - */ - @Explain(displayName = "keys") - public Map> getKeys() { - return keys; - } - - /** - * @param keys - * the keys to set - */ - public void setKeys(Map> keys) { - this.keys = keys; - } - - /** - * @return the position of the big table not in memory - */ - @Explain(displayName = "Position of Big Table") - public int getPosBigTable() { - return posBigTable; - } - - /** - * @param posBigTable - * the position of the big table not in memory - */ - public void setPosBigTable(int posBigTable) { - this.posBigTable = posBigTable; - } - - /** - * @return the keyTblDesc - */ - public TableDesc getKeyTblDesc() { - return keyTblDesc; - } - - /** - * @param keyTblDesc - * the keyTblDesc to set - */ - public void setKeyTblDesc(TableDesc keyTblDesc) { - this.keyTblDesc = keyTblDesc; - } - - /** - * @return the valueTblDescs - */ - public List getValueTblDescs() { - return valueTblDescs; - } - - /** - * @param valueTblDescs - * the valueTblDescs to set - */ - public void setValueTblDescs(List valueTblDescs) { - this.valueTblDescs = valueTblDescs; - } - - /** - * @return bigTableAlias - */ - public String getBigTableAlias() { - return bigTableAlias; - } - - /** - * @param bigTableAlias - */ - public void setBigTableAlias(String bigTableAlias) { - this.bigTableAlias = bigTableAlias; - } - - public LinkedHashMap>> getAliasBucketFileNameMapping() { - return aliasBucketFileNameMapping; - } - - public void setAliasBucketFileNameMapping( - LinkedHashMap>> aliasBucketFileNameMapping) { - this.aliasBucketFileNameMapping = aliasBucketFileNameMapping; - } - - public LinkedHashMap getBucketFileNameMapping() { - return bucketFileNameMapping; - } - - public void setBucketFileNameMapping(LinkedHashMap bucketFileNameMapping) { - this.bucketFileNameMapping = bucketFileNameMapping; - } -} Index: src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/plan/JDBMDummyDesc.java (working copy) @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; -/** - * JDBM Dummy Descriptor implementation. - * - */ -@Explain(displayName = "JDBMDummy Operator") -public class JDBMDummyDesc implements Serializable { - private TableDesc tbl; - - public TableDesc getTbl() { - return tbl; - } - - public void setTbl(TableDesc tbl) { - this.tbl = tbl; - } - -} Index: src/java/org/apache/hadoop/hive/ql/plan/HashTableDummyDesc.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/plan/HashTableDummyDesc.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/plan/HashTableDummyDesc.java (revision 0) @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +/** + * HashTable Dummy Descriptor implementation. + * + */ +@Explain(displayName = "HashTable Dummy Operator") +public class HashTableDummyDesc implements Serializable { + private TableDesc tbl; + + public TableDesc getTbl() { + return tbl; + } + + public void setTbl(TableDesc tbl) { + this.tbl = tbl; + } + +} Index: src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java (revision 0) +++ src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java (revision 0) @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +/** + * Map Join operator Descriptor implementation. + * + */ +@Explain(displayName = "HashTable Sink Operator") +public class HashTableSinkDesc extends JoinDesc implements Serializable { + private static final long serialVersionUID = 1L; + + + // used to handle skew join + private boolean handleSkewJoin = false; + private int skewKeyDefinition = -1; + private Map bigKeysDirMap; + private Map> smallKeysDirMap; + private Map skewKeysValuesTables; + + // alias to key mapping + private Map> exprs; + + // alias to filter mapping + private Map> filters; + + // used for create joinOutputObjectInspector + protected List outputColumnNames; + + // key:column output name, value:tag + private transient Map reversedExprs; + + // No outer join involved + protected boolean noOuterJoin; + + protected JoinCondDesc[] conds; + + protected Byte[] tagOrder; + private TableDesc keyTableDesc; + + + private Map> keys; + private TableDesc keyTblDesc; + private List valueTblDescs; + + private int posBigTable; + + private Map> retainList; + + private transient String bigTableAlias; + + private LinkedHashMap>> aliasBucketFileNameMapping; + private LinkedHashMap bucketFileNameMapping; + + public HashTableSinkDesc() { + bucketFileNameMapping = new LinkedHashMap(); + } + + public HashTableSinkDesc(MapJoinDesc clone) { + this.bigKeysDirMap = clone.getBigKeysDirMap(); + this.conds = clone.getConds(); + this.exprs= clone.getExprs(); + this.handleSkewJoin = clone.getHandleSkewJoin(); + this.keyTableDesc = clone.getKeyTableDesc(); + this.noOuterJoin = clone.getNoOuterJoin(); + this.outputColumnNames = clone.getOutputColumnNames(); + this.reversedExprs = clone.getReversedExprs(); + this.skewKeyDefinition = clone.getSkewKeyDefinition(); + this.skewKeysValuesTables = clone.getSkewKeysValuesTables(); + this.smallKeysDirMap = clone.getSmallKeysDirMap(); + this.tagOrder = clone.getTagOrder(); + this.filters = clone.getFilters(); + + this.keys = clone.getKeys(); + this.keyTblDesc = clone.getKeyTblDesc(); + this.valueTblDescs = clone.getValueTblDescs(); + this.posBigTable = clone.getPosBigTable(); + this.retainList = clone.getRetainList(); + this.bigTableAlias = clone.getBigTableAlias(); + this.aliasBucketFileNameMapping = clone.getAliasBucketFileNameMapping(); + this.bucketFileNameMapping = clone.getBucketFileNameMapping(); + } + + + private void initRetainExprList() { + retainList = new HashMap>(); + Set>> set = exprs.entrySet(); + Iterator>> setIter = set.iterator(); + while (setIter.hasNext()) { + Entry> current = setIter.next(); + List list = new ArrayList(); + for (int i = 0; i < current.getValue().size(); i++) { + list.add(i); + } + retainList.put(current.getKey(), list); + } + } + + public boolean isHandleSkewJoin() { + return handleSkewJoin; + } + + @Override + public void setHandleSkewJoin(boolean handleSkewJoin) { + this.handleSkewJoin = handleSkewJoin; + } + + @Override + public int getSkewKeyDefinition() { + return skewKeyDefinition; + } + + @Override + public void setSkewKeyDefinition(int skewKeyDefinition) { + this.skewKeyDefinition = skewKeyDefinition; + } + + @Override + public Map getBigKeysDirMap() { + return bigKeysDirMap; + } + + @Override + public void setBigKeysDirMap(Map bigKeysDirMap) { + this.bigKeysDirMap = bigKeysDirMap; + } + + @Override + public Map> getSmallKeysDirMap() { + return smallKeysDirMap; + } + + @Override + public void setSmallKeysDirMap(Map> smallKeysDirMap) { + this.smallKeysDirMap = smallKeysDirMap; + } + + @Override + public Map getSkewKeysValuesTables() { + return skewKeysValuesTables; + } + + @Override + public void setSkewKeysValuesTables(Map skewKeysValuesTables) { + this.skewKeysValuesTables = skewKeysValuesTables; + } + + @Override + public Map> getExprs() { + return exprs; + } + + @Override + public void setExprs(Map> exprs) { + this.exprs = exprs; + } + + @Override + public Map> getFilters() { + return filters; + } + + @Override + public void setFilters(Map> filters) { + this.filters = filters; + } + + @Override + public List getOutputColumnNames() { + return outputColumnNames; + } + + @Override + public void setOutputColumnNames(List outputColumnNames) { + this.outputColumnNames = outputColumnNames; + } + + @Override + public Map getReversedExprs() { + return reversedExprs; + } + + @Override + public void setReversedExprs(Map reversedExprs) { + this.reversedExprs = reversedExprs; + } + + @Override + public boolean isNoOuterJoin() { + return noOuterJoin; + } + + @Override + public void setNoOuterJoin(boolean noOuterJoin) { + this.noOuterJoin = noOuterJoin; + } + + @Override + public JoinCondDesc[] getConds() { + return conds; + } + + @Override + public void setConds(JoinCondDesc[] conds) { + this.conds = conds; + } + + @Override + public Byte[] getTagOrder() { + return tagOrder; + } + + @Override + public void setTagOrder(Byte[] tagOrder) { + this.tagOrder = tagOrder; + } + + @Override + public TableDesc getKeyTableDesc() { + return keyTableDesc; + } + + @Override + public void setKeyTableDesc(TableDesc keyTableDesc) { + this.keyTableDesc = keyTableDesc; + } + + + public Map> getRetainList() { + return retainList; + } + + public void setRetainList(Map> retainList) { + this.retainList = retainList; + } + + /** + * @return the keys + */ + @Explain(displayName = "keys") + public Map> getKeys() { + return keys; + } + + /** + * @param keys + * the keys to set + */ + public void setKeys(Map> keys) { + this.keys = keys; + } + + /** + * @return the position of the big table not in memory + */ + @Explain(displayName = "Position of Big Table") + public int getPosBigTable() { + return posBigTable; + } + + /** + * @param posBigTable + * the position of the big table not in memory + */ + public void setPosBigTable(int posBigTable) { + this.posBigTable = posBigTable; + } + + /** + * @return the keyTblDesc + */ + public TableDesc getKeyTblDesc() { + return keyTblDesc; + } + + /** + * @param keyTblDesc + * the keyTblDesc to set + */ + public void setKeyTblDesc(TableDesc keyTblDesc) { + this.keyTblDesc = keyTblDesc; + } + + /** + * @return the valueTblDescs + */ + public List getValueTblDescs() { + return valueTblDescs; + } + + /** + * @param valueTblDescs + * the valueTblDescs to set + */ + public void setValueTblDescs(List valueTblDescs) { + this.valueTblDescs = valueTblDescs; + } + + /** + * @return bigTableAlias + */ + public String getBigTableAlias() { + return bigTableAlias; + } + + /** + * @param bigTableAlias + */ + public void setBigTableAlias(String bigTableAlias) { + this.bigTableAlias = bigTableAlias; + } + + public LinkedHashMap>> getAliasBucketFileNameMapping() { + return aliasBucketFileNameMapping; + } + + public void setAliasBucketFileNameMapping( + LinkedHashMap>> aliasBucketFileNameMapping) { + this.aliasBucketFileNameMapping = aliasBucketFileNameMapping; + } + + public LinkedHashMap getBucketFileNameMapping() { + return bucketFileNameMapping; + } + + public void setBucketFileNameMapping(LinkedHashMap bucketFileNameMapping) { + this.bucketFileNameMapping = bucketFileNameMapping; + } +} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/SoftCache.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/SoftCache.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/SoftCache.java (working copy) @@ -1,328 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id - */ -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.lang.ref.Reference; -import java.lang.ref.ReferenceQueue; -import java.lang.ref.SoftReference; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; - -/** - * Wraps a deterministic cache policy with a Level-2 cache based on - * J2SE's {@link SoftReference soft references}. Soft references allow this - * cache to keep references to objects until the memory they occupy is required - * elsewhere. - *

- * Since the {@link CachePolicy} interface requires an event be fired when an - * object is evicted, and the event contains the actual object, this class - * cannot be a stand-alone implementation of CachePolicy. This - * limitation arises because Java References does not support notification - * before references are cleared; nor do they support reaching soft referents. - * Therefore, this wrapper cache aggressively notifies evictions: events are - * fired when the objects are evicted from the internal cache. Consequently, the - * soft cache may return a non-null object when get( ) is called, - * even if that object was said to have been evicted. - *

- * The current implementation uses a hash structure for its internal key to - * value mappings. - *

- * Note: this component's publicly exposed methods are not threadsafe; - * potentially concurrent code should synchronize on the cache instance. - * - * @author Dilum Ranatunga - * @version $Id: SoftCache.java,v 1.1 2003/11/01 13:29:27 dranatunga Exp $ - */ -public class SoftCache implements CachePolicy { - private static final int INITIAL_CAPACITY = 128; - private static final float DEFAULT_LOAD_FACTOR = 1.5f; - - private final ReferenceQueue _clearQueue = new ReferenceQueue(); - private final CachePolicy _internal; - private final Map _cacheMap; - - /** - * Creates a soft-reference based L2 cache with a {@link MRU} cache as the - * internal (L1) cache. The soft reference cache uses the default load - * capacity of 1.5f, which is intended to sacrifice some performance for - * space. This compromise is reasonable, since all {@link #get(Object) get( - * )s} first try the L1 cache anyway. The internal MRU is given a capacity of - * 128 elements. - */ - public SoftCache() { - this(new MRU(INITIAL_CAPACITY)); - } - - /** - * Creates a soft-reference based L2 cache wrapping the specified L1 cache. - * - * @param internal - * non null internal cache. - * @throws NullPointerException - * if the internal cache is null. - */ - public SoftCache(CachePolicy internal) throws NullPointerException { - this(DEFAULT_LOAD_FACTOR, internal); - } - - /** - * Creates a soft-reference based L2 cache wrapping the specified L1 cache. - * This constructor is somewhat implementation-specific, so users are - * encouraged to use {@link #SoftCache(CachePolicy)} instead. - * - * @param loadFactor - * load factor that the soft cache's hash structure should use. - * @param internal - * non null internal cache. - * @throws IllegalArgumentException - * if the load factor is nonpositive. - * @throws NullPointerException - * if the internal cache is null. - */ - public SoftCache(float loadFactor, CachePolicy internal) - throws IllegalArgumentException, NullPointerException { - if (internal == null) { - throw new NullPointerException("Internal cache cannot be null."); - } - _internal = internal; - _cacheMap = new HashMap(INITIAL_CAPACITY, loadFactor); - } - - /** - * Adds the specified value to the cache under the specified key. Note that - * the object is added to both this and the internal cache. - * - * @param key - * the (non-null) key to store the object under - * @param value - * the (non-null) object to place in the cache - * @throws CacheEvictionException - * exception that the internal cache would have experienced while - * evicting an object it currently cached. - */ - public void put(Object key, Object value) throws CacheEvictionException { - if (key == null) { - throw new IllegalArgumentException("key cannot be null."); - } else if (value == null) { - throw new IllegalArgumentException("value cannot be null."); - } - _internal.put(key, value); - removeClearedEntries(); - _cacheMap.put(key, new Entry(key, value, _clearQueue)); - } - - /** - * Gets the object cached under the specified key. - *

- * The cache is looked up in the following manner: - *

    - *
  1. The internal (L1) cache is checked. If the object is found, it is - * returned.
  2. - *
  3. This (L2) cache is checked. If the object is not found, then the caller - * is informed that the object is inaccessible.
  4. - *
  5. Since the object exists in L2, but not in L1, the object is readded to - * L1 using {@link CachePolicy#put(Object, Object)}.
  6. - *
  7. If the readding succeeds, the value is returned to caller.
  8. - *
  9. If a cache eviction exception is encountered instead, we remove the - * object from L2 and behave as if the object was inaccessible.
  10. - *
- * - * @param key - * the key that the object was stored under. - * @return the object stored under the key specified; null if the object is - * not (nolonger) accessible via this cache. - */ - public Object get(Object key) { - // first try the internal cache. - Object value = _internal.get(key); - if (value != null) { - return value; - } - // poll and remove cleared references. - removeClearedEntries(); - Entry entry = (Entry) _cacheMap.get(key); - if (entry == null) { // object is not in cache. - return null; - } - value = entry.getValue(); - if (value == null) { // object was in cache, but it was cleared. - return null; - } - // we have the object. so we try to re-insert it into internal cache - try { - _internal.put(key, value); - } catch (CacheEvictionException e) { - // if the internal cache causes a fuss, we kick the object out. - _cacheMap.remove(key); - return null; - } - return value; - } - - /** - * Removes any object stored under the key specified. Note that the object is - * removed from both this (L2) and the internal (L1) cache. - * - * @param key - * the key whose object should be removed - */ - public void remove(Object key) { - _cacheMap.remove(key); - _internal.remove(key); - } - - /** - * Removes all objects in this (L2) and its internal (L1) cache. - */ - public void removeAll() { - _cacheMap.clear(); - _internal.removeAll(); - } - - /** - * Gets all the objects stored by the internal (L1) cache. - * - * @return an enumeration of objects in internal cache. - */ - public Enumeration elements() { - return _internal.elements(); - } - - /** - * Adds the specified listener to this cache. Note that the events fired by - * this correspond to the internal cache's events. - * - * @param listener - * the (non-null) listener to add to this policy - * @throws IllegalArgumentException - * if listener is null. - */ - public void addListener(CachePolicyListener listener) - throws IllegalArgumentException { - _internal.addListener(listener); - } - - /** - * Removes a listener that was added earlier. - * - * @param listener - * the listener to remove. - */ - public void removeListener(CachePolicyListener listener) { - _internal.removeListener(listener); - } - - /** - * Cleans the mapping structure of any obsolete entries. This is usually - * called before insertions and lookups on the mapping structure. The runtime - * of this is usually very small, but it can be as expensive as n * log(n) if - * a large number of soft references were recently cleared. - */ - private final void removeClearedEntries() { - for (Reference r = _clearQueue.poll(); r != null; r = _clearQueue.poll()) { - Object key = ((Entry) r).getKey(); - _cacheMap.remove(key); - } - } - - /** - * Value objects we keep in the internal map. This contains the key in - * addition to the value, because polling for cleared references returns these - * instances, and having access to their corresponding keys drastically - * improves the performance of removing the pair from the map (see - * {@link SoftCache#removeClearedEntries()}.) - */ - private static class Entry extends SoftReference { - private final Object _key; - - /** - * Constructor that uses value as the soft reference's - * referent. - */ - public Entry(Object key, Object value, ReferenceQueue queue) { - super(value, queue); - _key = key; - } - - /** - * Gets the key - * - * @return the key associated with this value. - */ - final Object getKey() { - return _key; - } - - /** - * Gets the value - * - * @return the value; null if it is no longer accessible - */ - final Object getValue() { - return this.get(); - } - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ObjectBAComparator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ObjectBAComparator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ObjectBAComparator.java (working copy) @@ -1,179 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; -import java.io.Serializable; -import java.util.Comparator; - -/** - * Comparator for objects which have been serialized into byte arrays. In - * effect, it wraps another Comparator which compares object and provides - * transparent deserialization from byte array to object. - * - * @author Alex Boisvert - * @version $Id: ObjectBAComparator.java,v 1.1 2002/05/31 06:33:20 boisvert Exp - * $ - */ -public final class ObjectBAComparator implements Comparator, Serializable { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Wrapped comparator. - */ - private final Comparator _comparator; - - /** - * Construct an ObjectByteArrayComparator which wraps an Object Comparator. - * - * @param comparator - * Object comparator. - */ - public ObjectBAComparator(Comparator comparator) { - if (comparator == null) { - throw new IllegalArgumentException("Argument 'comparator' is null"); - } - - _comparator = comparator; - } - - /** - * Compare two objects. - * - * @param obj1 - * First object - * @param obj2 - * Second object - * @return 1 if obj1 > obj2, 0 if obj1 == obj2, -1 if obj1 < obj2 - */ - public int compare(Object obj1, Object obj2) { - if (obj1 == null) { - throw new IllegalArgumentException("Argument 'obj1' is null"); - } - - if (obj2 == null) { - throw new IllegalArgumentException("Argument 'obj2' is null"); - } - - try { - obj1 = Serialization.deserialize((byte[]) obj1); - obj2 = Serialization.deserialize((byte[]) obj2); - - return _comparator.compare(obj1, obj2); - } catch (IOException except) { - throw new WrappedRuntimeException(except); - } catch (ClassNotFoundException except) { - throw new WrappedRuntimeException(except); - } - } - - /** - * Compare two byte arrays. - */ - public static int compareByteArray(byte[] thisKey, byte[] otherKey) { - int len = Math.min(thisKey.length, otherKey.length); - - // compare the byte arrays - for (int i = 0; i < len; i++) { - if (thisKey[i] >= 0) { - if (otherKey[i] >= 0) { - // both positive - if (thisKey[i] < otherKey[i]) { - return -1; - } else if (thisKey[i] > otherKey[i]) { - return 1; - } - } else { - // otherKey is negative => greater (because MSB is 1) - return -1; - } - } else { - if (otherKey[i] >= 0) { - // thisKey is negative => greater (because MSB is 1) - return 1; - } else { - // both negative - if (thisKey[i] < otherKey[i]) { - return -1; - } else if (thisKey[i] > otherKey[i]) { - return 1; - } - } - } - } - if (thisKey.length == otherKey.length) { - return 0; - } - if (thisKey.length < otherKey.length) { - return -1; - } - return 1; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/MRU.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/MRU.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/MRU.java (working copy) @@ -1,345 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: MRU.java,v 1.8 2005/06/25 23:12:31 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.Vector; - -/** - * MRU - Most Recently Used cache policy. - * - * Methods are *not* synchronized, so no concurrent access is allowed. - * - * @author Alex Boisvert - * @version $Id: MRU.java,v 1.8 2005/06/25 23:12:31 doomdark Exp $ - */ -public class MRU implements CachePolicy { - - /** Cached object hashtable */ - Hashtable _hash = new Hashtable(); - - /** - * Maximum number of objects in the cache. - */ - int _max; - - /** - * Beginning of linked-list of cache elements. First entry is element which - * has been used least recently. - */ - CacheEntry _first; - - /** - * End of linked-list of cache elements. Last entry is element which has been - * used most recently. - */ - CacheEntry _last; - - /** - * Cache eviction listeners - */ - Vector listeners = new Vector(); - - /** - * Construct an MRU with a given maximum number of objects. - */ - public MRU(int max) { - if (max <= 0) { - throw new IllegalArgumentException( - "MRU cache must contain at least one entry"); - } - _max = max; - } - - /** - * Place an object in the cache. - */ - public void put(Object key, Object value) throws CacheEvictionException { - CacheEntry entry = (CacheEntry) _hash.get(key); - if (entry != null) { - entry.setValue(value); - touchEntry(entry); - } else { - - if (_hash.size() == _max) { - // purge and recycle entry - entry = purgeEntry(); - entry.setKey(key); - entry.setValue(value); - } else { - entry = new CacheEntry(key, value); - } - addEntry(entry); - _hash.put(entry.getKey(), entry); - } - } - - /** - * Obtain an object in the cache - */ - public Object get(Object key) { - CacheEntry entry = (CacheEntry) _hash.get(key); - if (entry != null) { - touchEntry(entry); - return entry.getValue(); - } else { - return null; - } - } - - /** - * Remove an object from the cache - */ - public void remove(Object key) { - CacheEntry entry = (CacheEntry) _hash.get(key); - if (entry != null) { - removeEntry(entry); - _hash.remove(entry.getKey()); - } - } - - /** - * Remove all objects from the cache - */ - public void removeAll() { - _hash = new Hashtable(); - _first = null; - _last = null; - } - - /** - * Enumerate elements' values in the cache - */ - public Enumeration elements() { - return new MRUEnumeration(_hash.elements()); - } - - /** - * Add a listener to this cache policy - * - * @param listener - * Listener to add to this policy - */ - public void addListener(CachePolicyListener listener) { - if (listener == null) { - throw new IllegalArgumentException("Cannot add null listener."); - } - if (!listeners.contains(listener)) { - listeners.addElement(listener); - } - } - - /** - * Remove a listener from this cache policy - * - * @param listener - * Listener to remove from this policy - */ - public void removeListener(CachePolicyListener listener) { - listeners.removeElement(listener); - } - - /** - * Add a CacheEntry. Entry goes at the end of the list. - */ - protected void addEntry(CacheEntry entry) { - if (_first == null) { - _first = entry; - _last = entry; - } else { - _last.setNext(entry); - entry.setPrevious(_last); - _last = entry; - } - } - - /** - * Remove a CacheEntry from linked list - */ - protected void removeEntry(CacheEntry entry) { - if (entry == _first) { - _first = entry.getNext(); - } - if (_last == entry) { - _last = entry.getPrevious(); - } - CacheEntry previous = entry.getPrevious(); - CacheEntry next = entry.getNext(); - if (previous != null) { - previous.setNext(next); - } - if (next != null) { - next.setPrevious(previous); - } - entry.setPrevious(null); - entry.setNext(null); - } - - /** - * Place entry at the end of linked list -- Most Recently Used - */ - protected void touchEntry(CacheEntry entry) { - if (_last == entry) { - return; - } - removeEntry(entry); - addEntry(entry); - } - - /** - * Purge least recently used object from the cache - * - * @return recyclable CacheEntry - */ - protected CacheEntry purgeEntry() throws CacheEvictionException { - CacheEntry entry = _first; - - // Notify policy listeners first. if any of them throw an - // eviction exception, then the internal data structure - // remains untouched. - CachePolicyListener listener; - for (int i = 0; i < listeners.size(); i++) { - listener = (CachePolicyListener) listeners.elementAt(i); - listener.cacheObjectEvicted(entry.getValue()); - } - - removeEntry(entry); - _hash.remove(entry.getKey()); - - entry.setValue(null); - return entry; - } - -} - -/** - * State information for cache entries. - */ -class CacheEntry { - private Object _key; - private Object _value; - - private CacheEntry _previous; - private CacheEntry _next; - - CacheEntry(Object key, Object value) { - _key = key; - _value = value; - } - - Object getKey() { - return _key; - } - - void setKey(Object obj) { - _key = obj; - } - - Object getValue() { - return _value; - } - - void setValue(Object obj) { - _value = obj; - } - - CacheEntry getPrevious() { - return _previous; - } - - void setPrevious(CacheEntry entry) { - _previous = entry; - } - - CacheEntry getNext() { - return _next; - } - - void setNext(CacheEntry entry) { - _next = entry; - } -} - -/** - * Enumeration wrapper to return actual user objects instead of CacheEntries. - */ -class MRUEnumeration implements Enumeration { - Enumeration _enum; - - MRUEnumeration(Enumeration enume) { - _enum = enume; - } - - public boolean hasMoreElements() { - return _enum.hasMoreElements(); - } - - public Object nextElement() { - CacheEntry entry = (CacheEntry) _enum.nextElement(); - return entry.getValue(); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/FastIterator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/FastIterator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/FastIterator.java (working copy) @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FastIterator.java,v 1.2 2003/10/21 15:43:58 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Fast and simple iterator. - * - * @version $Revision: 1.2 $ - * @author Alex Boisvert - */ -public abstract class FastIterator { - - /** - * Returns the next element in the interation. - * - * @return the next element in the iteration, or null if no more element. - */ - public abstract Object next() throws IterationException; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serializer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serializer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serializer.java (working copy) @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; -import java.io.Serializable; - -/** - * Interface used to provide a serialization mechanism other than a class' - * normal serialization. - * - * @author Alex Boisvert - * @version $Id: Serializer.java,v 1.1 2003/03/21 02:48:42 boisvert Exp $ - */ -public interface Serializer extends Serializable { - - /** - * Serialize the content of an object into a byte array. - * - * @param obj - * Object to serialize - * @return a byte array representing the object's state - */ - public byte[] serialize(Object obj) throws IOException; - - /** - * Deserialize the content of an object from a byte array. - * - * @param serialized - * Byte array representation of the object - * @return deserialized object - */ - public Object deserialize(byte[] serialized) throws IOException; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serialization.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serialization.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Serialization.java (working copy) @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -/** - * Serialization-related utility methods. - * - * @author Alex Boisvert - * @version $Id: Serialization.java,v 1.1 2002/05/31 06:33:20 boisvert Exp $ - */ -public final class Serialization { - - /** - * Serialize the object into a byte array. - */ - public static byte[] serialize(Object obj) throws IOException { - ByteArrayOutputStream baos; - ObjectOutputStream oos; - - baos = new ByteArrayOutputStream(); - oos = new ObjectOutputStream(baos); - oos.writeObject(obj); - oos.close(); - - return baos.toByteArray(); - } - - /** - * Deserialize an object from a byte array - */ - public static Object deserialize(byte[] buf) throws ClassNotFoundException, - IOException { - ByteArrayInputStream bais; - ObjectInputStream ois; - - bais = new ByteArrayInputStream(buf); - ois = new ObjectInputStream(bais); - return ois.readObject(); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IterationException.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IterationException.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IterationException.java (working copy) @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: IterationException.java,v 1.2 2003/09/21 15:47:00 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Iteration exception. - * - * @author Alex Boisvert - * @version $Revision: 1.2 $ - */ -public class IterationException extends WrappedRuntimeException { - - /** - * Construct a new iteration exception wrapping an underlying exception and - * providing a message. - * - * @param message - * The exception message - * @param except - * The underlying exception - */ - public IterationException(String message, Exception except) { - super(message, except); - } - - /** - * Construct a new iteration exception with a message. - * - * @param message - * The exception message - */ - public IterationException(String message) { - super(message, null); - } - - /** - * Construct a new iteration exception wrapping an underlying exception. - * - * @param except - * The underlying exception - */ - public IterationException(Exception except) { - super(except); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/DefaultSerializer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/DefaultSerializer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/DefaultSerializer.java (working copy) @@ -1,112 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; - -/** - * Default java serializer. - * - * @author Alex Boisvert - * @version $Id: DefaultSerializer.java,v 1.2 2003/09/21 15:47:00 boisvert Exp $ - */ -public class DefaultSerializer implements Serializer { - - public static final DefaultSerializer INSTANCE = new DefaultSerializer(); - - /** - * Construct a DefaultSerializer. - */ - public DefaultSerializer() { - // no op - } - - /** - * Serialize the content of an object into a byte array. - * - * @param obj - * Object to serialize - * @return a byte array representing the object's state - */ - public byte[] serialize(Object obj) throws IOException { - return Serialization.serialize(obj); - } - - /** - * Deserialize the content of an object from a byte array. - * - * @param serialized - * Byte array representation of the object - * @return deserialized object - */ - public Object deserialize(byte[] serialized) throws IOException { - try { - return Serialization.deserialize(serialized); - } catch (ClassNotFoundException except) { - throw new WrappedRuntimeException(except); - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CacheEvictionException.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CacheEvictionException.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CacheEvictionException.java (working copy) @@ -1,89 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: CacheEvictionException.java,v 1.4 2003/10/21 15:43:20 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Exception that occurs during eviction of an object in the cache. - * - * @author Alex Boisvert - * @version $Id: CacheEvictionException.java,v 1.4 2003/10/21 15:43:20 boisvert - * Exp $ - */ -public class CacheEvictionException extends Exception { - - /** - * Nested exception -- the original exception that occured, if any. - */ - protected Exception _nested; - - public CacheEvictionException(Exception nested) { - _nested = nested; - } - - public Exception getNestedException() { - return _nested; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicy.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicy.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicy.java (working copy) @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: CachePolicy.java,v 1.5 2003/11/01 13:25:02 dranatunga Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.util.Enumeration; - -/** - * CachePolicity is an abstraction for different cache policies. (ie. MRU, - * time-based, soft-refs, ...) - * - * @author Alex Boisvert - * @author Dilum Ranatunga - * @version $Id: CachePolicy.java,v 1.5 2003/11/01 13:25:02 dranatunga Exp $ - */ -public interface CachePolicy { - - /** - * Place an object in the cache. If the cache does not currently contain an - * object for the key specified, this mapping is added. If an object currently - * exists under the specified key, the current object is replaced with the new - * object. - *

- * If the changes to the cache cause the eviction of any objects - * stored under other key(s), events corresponding to the - * evictions are fired for each object. If an event listener is unable to - * handle the eviction, and throws a cache eviction exception, that exception - * is propagated to the caller. If such an exception is thrown, the cache - * itself should be left as it was before the put() operation was - * invoked: the the object whose eviction failed is still in the cache, and - * the new insertion or modification is reverted. - * - * @param key - * key for the cached object - * @param value - * the cached object - * @throws CacheEvictionException - * propagated if, while evicting objects to make room for new - * object, an eviction listener encountered this problem. - */ - public void put(Object key, Object value) throws CacheEvictionException; - - /** - * Obtain the object stored under the key specified. - * - * @param key - * key the object was cached under - * @return the object if it is still in the cache, null otherwise. - */ - public Object get(Object key); - - /** - * Remove the object stored under the key specified. Note that since eviction - * notices are only fired when objects under different keys - * are evicted, no event is fired for any object stored under this key (see - * {@link #put(Object, Object) put( )}). - * - * @param key - * key the object was stored in the cache under. - */ - public void remove(Object key); - - /** - * Remove all objects from the cache. Consistent with {@link #remove(Object) - * remove( )}, no eviction notices are fired. - */ - public void removeAll(); - - /** - * Enumerate through the objects currently in the cache. - */ - public Enumeration elements(); - - /** - * Add a listener to this cache policy. - *

- * If this cache policy already contains a listener that is equal to the one - * being added, this call has no effect. - * - * @param listener - * the (non-null) listener to add to this policy - * @throws IllegalArgumentException - * if listener is null. - */ - public void addListener(CachePolicyListener listener) - throws IllegalArgumentException; - - /** - * Remove a listener from this cache policy. The listener is found using - * object equality, not identity. - * - * @param listener - * the listener to remove from this policy - */ - public void removeListener(CachePolicyListener listener); - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/WrappedRuntimeException.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/WrappedRuntimeException.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/WrappedRuntimeException.java (working copy) @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.PrintStream; -import java.io.PrintWriter; - -/** - * A run-time exception that wraps another exception. The printed stack trace - * will be that of the wrapped exception. - * - * @author Alex Boisvert - * @version $Id: WrappedRuntimeException.java,v 1.1 2002/05/31 06:33:20 boisvert - * Exp $ - */ -public class WrappedRuntimeException extends RuntimeException { - - /** - * The underlying exception. - */ - private final Exception _except; - - /** - * Constructs a new runtime exception based on a checked exception. - * - * @param message - * The error message - * @param except - * The checked exception - */ - public WrappedRuntimeException(String message, Exception except) { - super(message == null ? "No message available" : message); - - if (except instanceof WrappedRuntimeException - && ((WrappedRuntimeException) except)._except != null) { - _except = ((WrappedRuntimeException) except)._except; - } else { - _except = except; - } - } - - /** - * Constructs a new runtime exception based on a checked exception. - * - * @param except - * The checked exception - */ - public WrappedRuntimeException(Exception except) { - super( - except == null || except.getMessage() == null ? "No message available" - : except.getMessage()); - - if (except instanceof WrappedRuntimeException - && ((WrappedRuntimeException) except)._except != null) { - _except = ((WrappedRuntimeException) except)._except; - } else { - _except = except; - } - } - - /** - * Returns the exception wrapped by this runtime exception. - * - * @return The exception wrapped by this runtime exception - */ - public Exception getException() { - return _except; - } - - @Override - public void printStackTrace() { - if (_except == null) { - super.printStackTrace(); - } else { - _except.printStackTrace(); - } - } - - @Override - public void printStackTrace(PrintStream stream) { - if (_except == null) { - super.printStackTrace(stream); - } else { - _except.printStackTrace(stream); - } - } - - @Override - public void printStackTrace(PrintWriter writer) { - if (_except == null) { - super.printStackTrace(writer); - } else { - _except.printStackTrace(writer); - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Conversion.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Conversion.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Conversion.java (working copy) @@ -1,211 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Miscelaneous conversion utility methods. - * - * @author Alex Boisvert - * @version $Id: Conversion.java,v 1.3 2002/05/31 06:33:20 boisvert Exp $ - */ -public class Conversion { - - /** - * Convert a string into a byte array. - */ - public static byte[] convertToByteArray(String s) { - try { - // see the following page for character encoding - // http://java.sun.com/products/jdk/1.1/docs/guide/intl/encoding.doc.html - return s.getBytes("UTF8"); - } catch (java.io.UnsupportedEncodingException uee) { - uee.printStackTrace(); - throw new Error("Platform doesn't support UTF8 encoding"); - } - } - - /** - * Convert a byte into a byte array. - */ - public static byte[] convertToByteArray(byte n) { - n = (byte) (n ^ ((byte) 0x80)); // flip MSB because "byte" is signed - return new byte[] {n}; - } - - /** - * Convert a short into a byte array. - */ - public static byte[] convertToByteArray(short n) { - n = (short) (n ^ ((short) 0x8000)); // flip MSB because "short" is signed - byte[] key = new byte[2]; - pack2(key, 0, n); - return key; - } - - /** - * Convert an int into a byte array. - */ - public static byte[] convertToByteArray(int n) { - n = (n ^ 0x80000000); // flip MSB because "int" is signed - byte[] key = new byte[4]; - pack4(key, 0, n); - return key; - } - - /** - * Convert a long into a byte array. - */ - public static byte[] convertToByteArray(long n) { - n = (n ^ 0x8000000000000000L); // flip MSB because "long" is signed - byte[] key = new byte[8]; - pack8(key, 0, n); - return key; - } - - /** - * Convert a byte array (encoded as UTF-8) into a String - */ - public static String convertToString(byte[] buf) { - try { - // see the following page for character encoding - // http://java.sun.com/products/jdk/1.1/docs/guide/intl/encoding.doc.html - return new String(buf, "UTF8"); - } catch (java.io.UnsupportedEncodingException uee) { - uee.printStackTrace(); - throw new Error("Platform doesn't support UTF8 encoding"); - } - } - - /** - * Convert a byte array into an integer (signed 32-bit) value. - */ - public static int convertToInt(byte[] buf) { - int value = unpack4(buf, 0); - value = (value ^ 0x80000000); // flip MSB because "int" is signed - return value; - } - - /** - * Convert a byte array into a long (signed 64-bit) value. - */ - public static long convertToLong(byte[] buf) { - long value = ((long) unpack4(buf, 0) << 32) - + (unpack4(buf, 4) & 0xFFFFFFFFL); - value = (value ^ 0x8000000000000000L); // flip MSB because "long" is signed - return value; - } - - static int unpack4(byte[] buf, int offset) { - int value = (buf[offset] << 24) | ((buf[offset + 1] << 16) & 0x00FF0000) - | ((buf[offset + 2] << 8) & 0x0000FF00) - | ((buf[offset + 3] << 0) & 0x000000FF); - - return value; - } - - static final void pack2(byte[] data, int offs, int val) { - data[offs++] = (byte) (val >> 8); - data[offs++] = (byte) val; - } - - static final void pack4(byte[] data, int offs, int val) { - data[offs++] = (byte) (val >> 24); - data[offs++] = (byte) (val >> 16); - data[offs++] = (byte) (val >> 8); - data[offs++] = (byte) val; - } - - static final void pack8(byte[] data, int offs, long val) { - pack4(data, 0, (int) (val >> 32)); - pack4(data, 4, (int) val); - } - - /** - * Test static methods - */ - public static void main(String[] args) { - byte[] buf; - - buf = convertToByteArray(5); - System.out.println("int value of 5 is: " + convertToInt(buf)); - - buf = convertToByteArray(-1); - System.out.println("int value of -1 is: " + convertToInt(buf)); - - buf = convertToByteArray(22111000); - System.out.println("int value of 22111000 is: " + convertToInt(buf)); - - buf = convertToByteArray(5L); - System.out.println("long value of 5 is: " + convertToLong(buf)); - - buf = convertToByteArray(-1L); - System.out.println("long value of -1 is: " + convertToLong(buf)); - - buf = convertToByteArray(1112223334445556667L); - System.out.println("long value of 1112223334445556667 is: " - + convertToLong(buf)); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerComparator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerComparator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerComparator.java (working copy) @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.Serializable; -import java.util.Comparator; - -/** - * Comparator for Integer objects. - * - * @author Christof Dallermassl - * @version $Id: IntegerComparator.java,v 1.2 2002/05/31 06:33:20 boisvert Exp $ - */ -public final class IntegerComparator implements Comparator, Serializable { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Compare two objects. - * - * @param obj1 - * First object - * @param obj2 - * Second object - * @return a positive integer if obj1 > obj2, 0 if obj1 == obj2, and a - * negative integer if obj1 < obj2 - */ - public int compare(Object obj1, Object obj2) { - if (obj1 == obj2) { - return 0; - } - - if (obj1 == null) { - throw new IllegalArgumentException("Argument 'obj1' is null"); - } - - if (obj2 == null) { - throw new IllegalArgumentException("Argument 'obj2' is null"); - } - - // complicated to avoid usage of Integer.compareTo, as this - // method is Java 1.2 only! - int int1 = ((Integer) obj1).intValue(); - int int2 = ((Integer) obj2).intValue(); - if (int1 == int2) { - return 0; - } - - if (int1 < int2) { - return -1; - } else { - return 1; - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerSerializer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerSerializer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/IntegerSerializer.java (working copy) @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; - -/** - * Optimized serializer for integers. - * - * @author Alex Boisvert - * @version $Id: IntegerSerializer.java,v 1.2 2003/09/21 15:47:00 boisvert Exp $ - */ -public class IntegerSerializer implements Serializer { - - public static final IntegerSerializer INSTANCE = new IntegerSerializer(); - - /** - * Construct an IntegerSerializer. - */ - public IntegerSerializer() { - // no op - } - - /** - * Serialize the content of an object into a byte array. - * - * @param obj - * Object to serialize - * @return a byte array representing the object's state - */ - public byte[] serialize(Object obj) throws IOException { - Integer number = (Integer) obj; - return Conversion.convertToByteArray(number.intValue()); - } - - /** - * Deserialize the content of an object from a byte array. - * - * @param serialized - * Byte array representation of the object - * @return deserialized object - */ - public Object deserialize(byte[] serialized) throws IOException { - int number = Conversion.convertToInt(serialized); - return new Integer(number); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongComparator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongComparator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongComparator.java (working copy) @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.Serializable; -import java.util.Comparator; - -/** - * Comparator for java.lang.Long objects. - * - * @author Alex Boisvert - * @version $Id: LongComparator.java,v 1.4 2002/05/31 06:33:20 boisvert Exp $ - */ -public final class LongComparator implements Comparator, Serializable { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Compare two objects. - * - * @param obj1 - * First object - * @param obj2 - * Second object - * @return a positive integer if obj1 > obj2, 0 if obj1 == obj2, and a - * negative integer if obj1 < obj2 - */ - public int compare(Object obj1, Object obj2) { - if (obj1 == null) { - throw new IllegalArgumentException("Argument 'obj1' is null"); - } - - if (obj2 == null) { - throw new IllegalArgumentException("Argument 'obj2' is null"); - } - - long l1 = ((Long) obj1).longValue(); - long l2 = ((Long) obj2).longValue(); - - if (l1 > l2) { - return 1; - } else if (l1 == l2) { - return 0; - } else { - return -1; - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongSerializer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongSerializer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/LongSerializer.java (working copy) @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; - -/** - * Optimized serializer for long integers. - * - * @author Alex Boisvert - * @version $Id: LongSerializer.java,v 1.2 2003/09/21 15:47:00 boisvert Exp $ - */ -public class LongSerializer implements Serializer { - - public static final LongSerializer INSTANCE = new LongSerializer(); - - /** - * Construct a LongSerializer. - */ - public LongSerializer() { - // no op - } - - /** - * Serialize the content of an object into a byte array. - * - * @param obj - * Object to serialize - * @return a byte array representing the object's state - */ - public byte[] serialize(Object obj) throws IOException { - Long number = (Long) obj; - return Conversion.convertToByteArray(number.longValue()); - } - - /** - * Deserialize the content of an object from a byte array. - * - * @param serialized - * Byte array representation of the object - * @return deserialized object - */ - public Object deserialize(byte[] serialized) throws IOException { - long number = Conversion.convertToLong(serialized); - return new Long(number); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicyListener.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicyListener.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/CachePolicyListener.java (working copy) @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: CachePolicyListener.java,v 1.3 2003/11/01 13:25:41 dranatunga Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Callback interface between {@link CachePolicy} and a Cache implementation to - * notify about cached object eviction. - *

- * Note that CachePolicy implementations typically use - * object equality when removing listeners, so concrete implementations - * of this interface should also pay attention to their - * {@link Object#equals(Object)} and {@link Object#hashCode()} methods. - * - * @author Alex Boisvert - * @version $Id: CachePolicyListener.java,v 1.3 2003/11/01 13:25:41 dranatunga - * Exp $ - */ -public interface CachePolicyListener { - - /** - * Notification that the cache this listener is attached to is evicting the - * object indicated. - * - * @param obj - * object being evited from cache - * @throws CacheEvictionException - * if this listener encountered problems while preparing for the - * specified object's eviction. For example, a listener may try to - * persist the object to disk, and encounter an - * IOException. - */ - public void cacheObjectEvicted(Object obj) throws CacheEvictionException; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Tuple.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Tuple.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/Tuple.java (working copy) @@ -1,133 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -/** - * Tuple consisting of a key-value pair. - * - * @author Alex Boisvert - * @version $Id: Tuple.java,v 1.2 2001/05/19 14:02:00 boisvert Exp $ - */ -public final class Tuple { - - /** - * Key - */ - private Object _key; - - /** - * Value - */ - private Object _value; - - /** - * Construct an empty Tuple. - */ - public Tuple() { - // empty - } - - /** - * Construct a Tuple. - * - * @param key - * The key. - * @param value - * The value. - */ - public Tuple(Object key, Object value) { - _key = key; - _value = value; - } - - /** - * Get the key. - */ - public Object getKey() { - return _key; - } - - /** - * Set the key. - */ - public void setKey(Object key) { - _key = key; - } - - /** - * Get the value. - */ - public Object getValue() { - return _value; - } - - /** - * Set the value. - */ - public void setValue(Object value) { - _value = value; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArrayComparator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArrayComparator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArrayComparator.java (working copy) @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.Serializable; -import java.util.Comparator; - -/** - * Comparator for byte arrays. - * - * @author Alex Boisvert - * @version $Id: ByteArrayComparator.java,v 1.4 2002/05/31 06:33:20 boisvert Exp - * $ - */ -public final class ByteArrayComparator implements Comparator, Serializable { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Compare two objects. - * - * @param obj1 - * First object - * @param obj2 - * Second object - * @return a positive integer if obj1 > obj2, 0 if obj1 == obj2, and a - * negative integer if obj1 < obj2 - */ - public int compare(Object obj1, Object obj2) { - if (obj1 == null) { - throw new IllegalArgumentException("Argument 'obj1' is null"); - } - - if (obj2 == null) { - throw new IllegalArgumentException("Argument 'obj2' is null"); - } - - return compareByteArray((byte[]) obj1, (byte[]) obj2); - } - - /** - * Compare two byte arrays. - */ - public static int compareByteArray(byte[] thisKey, byte[] otherKey) { - int len = Math.min(thisKey.length, otherKey.length); - - // compare the byte arrays - for (int i = 0; i < len; i++) { - if (thisKey[i] >= 0) { - if (otherKey[i] >= 0) { - // both positive - if (thisKey[i] < otherKey[i]) { - return -1; - } else if (thisKey[i] > otherKey[i]) { - return 1; - } - } else { - // otherKey is negative => greater (because MSB is 1) - return -1; - } - } else { - if (otherKey[i] >= 0) { - // thisKey is negative => greater (because MSB is 1) - return 1; - } else { - // both negative - if (thisKey[i] < otherKey[i]) { - return -1; - } else if (thisKey[i] > otherKey[i]) { - return 1; - } - } - } - } - if (thisKey.length == otherKey.length) { - return 0; - } - if (thisKey.length < otherKey.length) { - return -1; - } - return 1; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArraySerializer.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArraySerializer.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/ByteArraySerializer.java (working copy) @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; - -/** - * Serializer for byte arrays -- simple returns the byte array itself. No actual - * serialization is performed. - * - * @author Alex Boisvert - * @version $Id: ByteArraySerializer.java,v 1.1 2003/03/21 02:48:42 boisvert Exp - * $ - */ -public final class ByteArraySerializer implements Serializer { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Static instance. - */ - public static final ByteArraySerializer INSTANCE = new ByteArraySerializer(); - - /** - * Serialize the content of an object into a byte array. - * - * @param obj - * Object to serialize - * @return a byte array representing the object's state - * - */ - public byte[] serialize(Object obj) throws IOException { - return (byte[]) obj; - } - - /** - * Deserialize the content of an object from a byte array. - * - * @param serialized - * Byte array representation of the object - * @return deserialized object - * - */ - public Object deserialize(byte[] serialized) throws IOException { - return serialized; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/TupleBrowser.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/TupleBrowser.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/TupleBrowser.java (working copy) @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.IOException; - -/** - * Browser to traverse a collection of tuples. The browser allows for forward - * and reverse order traversal. - * - * @author Alex Boisvert - * @version $Id: TupleBrowser.java,v 1.2 2001/05/19 14:02:00 boisvert Exp $ - */ -public abstract class TupleBrowser { - - /** - * Get the next tuple. - * - * @param tuple - * Tuple into which values are copied. - * @return True if values have been copied in tuple, or false if there is no - * next tuple. - */ - public abstract boolean getNext(Tuple tuple) throws IOException; - - /** - * Get the previous tuple. - * - * @param tuple - * Tuple into which values are copied. - * @return True if values have been copied in tuple, or false if there is no - * previous tuple. - */ - public abstract boolean getPrevious(Tuple tuple) throws IOException; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/StringComparator.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/StringComparator.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/helper/StringComparator.java (working copy) @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2001 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.helper; - -import java.io.Serializable; -import java.util.Comparator; - -/** - * Comparator for String objects. Delegates to String.compareTo(). - * - * @author Alex Boisvert - * @version $Id: StringComparator.java,v 1.5 2005/06/25 23:12:31 doomdark Exp $ - */ -public final class StringComparator implements Comparator, Serializable { - - /** - * Version id for serialization. - */ - final static long serialVersionUID = 1L; - - /** - * Compare two objects. - * - * @param obj1 - * First object - * @param obj2 - * Second object - * @return a positive integer if obj1 > obj2, 0 if obj1 == obj2, and a - * negative integer if obj1 < obj2 - */ - public int compare(Object obj1, Object obj2) { - if (obj1 == null) { - throw new IllegalArgumentException("Argument 'obj1' is null"); - } - - if (obj2 == null) { - throw new IllegalArgumentException("Argument 'obj2' is null"); - } - - return ((String) obj1).compareTo((String) obj2); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerFactory.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerFactory.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerFactory.java (working copy) @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordManagerFactory.java,v 1.2 2005/06/25 23:12:31 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; - -/** - * This is the factory class to use for instantiating {@link RecordManager} - * instances. - * - * @author Alex Boisvert - * @author Cees de Groot - * @version $Id: RecordManagerFactory.java,v 1.2 2005/06/25 23:12:31 doomdark - * Exp $ - */ -public final class RecordManagerFactory { - - /** - * Create a record manager. - * - * @param name - * Name of the record file. - * @throws IOException - * if an I/O related exception occurs while creating or opening the - * record manager. - * @throws UnsupportedOperationException - * if some options are not supported by the implementation. - * @throws IllegalArgumentException - * if some options are invalid. - */ - public static RecordManager createRecordManager(String name) - throws IOException { - return createRecordManager(name, new Properties()); - } - - /** - * Create a record manager. - * - * @param name - * Name of the record file. - * @param options - * Record manager options. - * @throws IOException - * if an I/O related exception occurs while creating or opening the - * record manager. - * @throws UnsupportedOperationException - * if some options are not supported by the implementation. - * @throws IllegalArgumentException - * if some options are invalid. - */ - public static RecordManager createRecordManager(String name, - Properties options) throws IOException { - RecordManagerProvider factory = getFactory(options); - return factory.createRecordManager(name, options); - } - - public static RecordManager createRecordManager(File file, Properties options) - throws IOException { - RecordManagerProvider factory = getFactory(options); - return factory.createRecordManager(file, options); - } - - private static RecordManagerProvider getFactory(Properties options) { - String provider; - Class clazz; - RecordManagerProvider factory; - - provider = options.getProperty(RecordManagerOptions.PROVIDER_FACTORY, - "org.apache.hadoop.hive.ql.util.jdbm.recman.Provider"); - - try { - clazz = Class.forName(provider); - factory = (RecordManagerProvider) clazz.newInstance(); - } catch (Exception except) { - throw new IllegalArgumentException("Invalid record manager provider: " - + provider + "\n[" + except.getClass().getName() + ": " - + except.getMessage() + "]"); - } - return factory; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerProvider.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerProvider.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerProvider.java (working copy) @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordManagerProvider.java,v 1.2 2005/06/25 23:12:31 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; - -/** - * Provider of RecordManager implementation. Classes implementing this interface - * act as a factory to provide implementations of RecordManager. - * - * @author Alex Boisvert - * @version $Id: RecordManagerProvider.java,v 1.2 2005/06/25 23:12:31 doomdark - * Exp $ - */ -public interface RecordManagerProvider { - - /** - * Create a record manager. - * - * @param filename - * Base filename of the record file. - * @param options - * Record manager options. - * @throws IOException - * if an I/O related exception occurs while creating or opening the - * record manager. - * @throws UnsupportedOperationException - * if some options are not supported by the implementation. - * @throws IllegalArgumentException - * if some options are invalid. - */ - public RecordManager createRecordManager(String filename, Properties options) - throws IOException; - - public RecordManager createRecordManager(File file, Properties options) - throws IOException; -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerOptions.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerOptions.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManagerOptions.java (working copy) @@ -1,138 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordManagerOptions.java,v 1.1 2002/05/31 06:33:20 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm; - -/** - * Standard options for RecordManager. - * - * @author Alex Boisvert - * @author Cees de Groot - * @version $Id: RecordManagerOptions.java,v 1.1 2002/05/31 06:33:20 boisvert - * Exp $ - */ -public class RecordManagerOptions { - - /** - * Option to create a thread-safe record manager. - */ - public static final String PROVIDER_FACTORY = "jdbm.provider"; - - /** - * Option to create a thread-safe record manager. - */ - public static final String THREAD_SAFE = "jdbm.threadSafe"; - - /** - * Option to automatically commit data after each operation. - */ - public static final String AUTO_COMMIT = "jdbm.autoCommit"; - - /** - * Option to disable transaction (to increase performance at the cost of - * potential data loss). - */ - public static final String DISABLE_TRANSACTIONS = "jdbm.disableTransactions"; - - /** - * Cache type. - */ - public static final String CACHE_TYPE = "jdbm.cache.type"; - - /** - * Cache size (when applicable) - */ - public static final String CACHE_SIZE = "jdbm.cache.size"; - - /** - * Use normal (strong) object references for the record cache. - */ - public static final String NORMAL_CACHE = "normal"; - - /** - * Use soft references {$link java.lang.ref.SoftReference} for the record - * cache instead of the default normal object references. - *

- * Soft references are cleared at the discretion of the garbage collector in - * response to memory demand. - */ - public static final String SOFT_REF_CACHE = "soft"; - - /** - * Use weak references {$link java.lang.ref.WeakReference} for the record - * cache instead of the default normal object references. - *

- * Weak references do not prevent their referents from being made finalizable, - * finalized, and then reclaimed. - */ - public static final String WEAK_REF_CACHE = "weak"; - - /** - * Disable cache. - */ - public static final String NO_CACHE = "nocache"; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordCache.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordCache.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordCache.java (working copy) @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordCache.java,v 1.2 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This interface is used for synchronization. - *

- * RecordManager ensures that the cache has the up-to-date information by way of - * an invalidation protocol. - */ -public interface RecordCache { - - /** - * Notification to flush content related to a given record. - */ - public void flush(long recid) throws IOException; - - /** - * Notification to flush data all of records. - */ - public void flushAll() throws IOException; - - /** - * Notification to invalidate content related to given record. - */ - public void invalidate(long recid) throws IOException; - - /** - * Notification to invalidate content of all records. - */ - public void invalidateAll() throws IOException; - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BaseRecordManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BaseRecordManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BaseRecordManager.java (working copy) @@ -1,476 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: BaseRecordManager.java,v 1.8 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.helper.DefaultSerializer; -import org.apache.hadoop.hive.ql.util.jdbm.helper.Serializer; - -/** - * This class manages records, which are uninterpreted blobs of data. The set of - * operations is simple and straightforward: you communicate with the class - * using long "rowids" and byte[] data blocks. Rowids are returned on inserts - * and you can stash them away someplace safe to be able to get back to them. - * Data blocks can be as long as you wish, and may have lengths different from - * the original when updating. - *

- * Operations are synchronized, so that only one of them will happen - * concurrently even if you hammer away from multiple threads. Operations are - * made atomic by keeping a transaction log which is recovered after a crash, so - * the operations specified by this interface all have ACID properties. - *

- * You identify a file by just the name. The package attaches .db for - * the database file, and .lg for the transaction log. The transaction - * log is synchronized regularly and then restarted, so don't worry if you see - * the size going up and down. - * - * @author Alex Boisvert - * @author Cees de Groot - * @version $Id: BaseRecordManager.java,v 1.8 2005/06/25 23:12:32 doomdark Exp $ - */ -public final class BaseRecordManager implements RecordManager { - - /** - * Underlying record file. - */ - private RecordFile _file; - - /** - * Physical row identifier manager. - */ - private final PhysicalRowIdManager _physMgr; - - /** - * Logigal to Physical row identifier manager. - */ - private final LogicalRowIdManager _logMgr; - - /** - * Page manager. - */ - private PageManager _pageman; - - /** - * Reserved slot for name directory. - */ - public static final int NAME_DIRECTORY_ROOT = 0; - - /** - * Static debugging flag - */ - public static final boolean DEBUG = false; - - /** - * Directory of named JDBMHashtables. This directory is a persistent - * directory, stored as a Hashtable. It can be retrived by using the - * NAME_DIRECTORY_ROOT. - */ - private Map _nameDirectory; - - /** - * Creates a record manager for the indicated file - * - * @throws IOException - * when the file cannot be opened or is not a valid file - * content-wise. - */ - public BaseRecordManager(String filename) throws IOException { - _file = new RecordFile(filename); - _pageman = new PageManager(_file); - _physMgr = new PhysicalRowIdManager(_file, _pageman); - _logMgr = new LogicalRowIdManager(_file, _pageman); - } - - /** - * Creates a record manager for the indicated file - * - * @throws IOException - * when the file cannot be opened or is not a valid file - * content-wise. - */ - public BaseRecordManager(File file) throws IOException { - _file = new RecordFile(file); - _pageman = new PageManager(_file); - _physMgr = new PhysicalRowIdManager(_file, _pageman); - _logMgr = new LogicalRowIdManager(_file, _pageman); - } - - /** - * Get the underlying Transaction Manager - */ - public synchronized TransactionManager getTransactionManager() { - checkIfClosed(); - - return _file.txnMgr; - } - - /** - * Switches off transactioning for the record manager. This means that a) a - * transaction log is not kept, and b) writes aren't synch'ed after every - * update. This is useful when batch inserting into a new database. - *

- * Only call this method directly after opening the file, otherwise the - * results will be undefined. - */ - public synchronized void disableTransactions() { - checkIfClosed(); - - _file.disableTransactions(); - } - - /** - * Closes the record manager. - * - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void close() throws IOException { - checkIfClosed(); - - _pageman.close(); - _pageman = null; - - _file.close(); - _file = null; - } - - /** - * Inserts a new record using standard java object serialization. - * - * @param obj - * the object for the new record. - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public long insert(Object obj) throws IOException { - return insert(obj, DefaultSerializer.INSTANCE); - } - - /** - * Inserts a new record using a custom serializer. - * - * @param obj - * the object for the new record. - * @param serializer - * a custom serializer - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized long insert(Object obj, Serializer serializer) - throws IOException { - byte[] data; - long recid; - Location physRowId; - - checkIfClosed(); - - data = serializer.serialize(obj); - physRowId = _physMgr.insert(data, 0, data.length); - recid = _logMgr.insert(physRowId).toLong(); - if (DEBUG) { - System.out.println("BaseRecordManager.insert() recid " + recid - + " length " + data.length); - } - return recid; - } - - /** - * Deletes a record. - * - * @param recid - * the rowid for the record that should be deleted. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void delete(long recid) throws IOException { - checkIfClosed(); - if (recid <= 0) { - throw new IllegalArgumentException("Argument 'recid' is invalid: " - + recid); - } - - if (DEBUG) { - System.out.println("BaseRecordManager.delete() recid " + recid); - } - - Location logRowId = new Location(recid); - Location physRowId = _logMgr.fetch(logRowId); - _physMgr.delete(physRowId); - _logMgr.delete(logRowId); - } - - /** - * Updates a record using standard java object serialization. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public void update(long recid, Object obj) throws IOException { - update(recid, obj, DefaultSerializer.INSTANCE); - } - - /** - * Updates a record using a custom serializer. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @param serializer - * a custom serializer - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void update(long recid, Object obj, Serializer serializer) - throws IOException { - checkIfClosed(); - if (recid <= 0) { - throw new IllegalArgumentException("Argument 'recid' is invalid: " - + recid); - } - - Location logRecid = new Location(recid); - Location physRecid = _logMgr.fetch(logRecid); - - byte[] data = serializer.serialize(obj); - if (DEBUG) { - System.out.println("BaseRecordManager.update() recid " + recid - + " length " + data.length); - } - - Location newRecid = _physMgr.update(physRecid, data, 0, data.length); - if (!newRecid.equals(physRecid)) { - _logMgr.update(logRecid, newRecid); - } - } - - /** - * Fetches a record using standard java object serialization. - * - * @param recid - * the recid for the record that must be fetched. - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public Object fetch(long recid) throws IOException { - return fetch(recid, DefaultSerializer.INSTANCE); - } - - /** - * Fetches a record using a custom serializer. - * - * @param recid - * the recid for the record that must be fetched. - * @param serializer - * a custom serializer - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized Object fetch(long recid, Serializer serializer) - throws IOException { - byte[] data; - - checkIfClosed(); - if (recid <= 0) { - throw new IllegalArgumentException("Argument 'recid' is invalid: " - + recid); - } - data = _physMgr.fetch(_logMgr.fetch(new Location(recid))); - if (DEBUG) { - System.out.println("BaseRecordManager.fetch() recid " + recid - + " length " + data.length); - } - return serializer.deserialize(data); - } - - /** - * Returns the number of slots available for "root" rowids. These slots can be - * used to store special rowids, like rowids that point to other rowids. Root - * rowids are useful for bootstrapping access to a set of data. - */ - public int getRootCount() { - return FileHeader.NROOTS; - } - - /** - * Returns the indicated root rowid. - * - * @see #getRootCount - */ - public synchronized long getRoot(int id) throws IOException { - checkIfClosed(); - - return _pageman.getFileHeader().getRoot(id); - } - - /** - * Sets the indicated root rowid. - * - * @see #getRootCount - */ - public synchronized void setRoot(int id, long rowid) throws IOException { - checkIfClosed(); - - _pageman.getFileHeader().setRoot(id, rowid); - } - - /** - * Obtain the record id of a named object. Returns 0 if named object doesn't - * exist. - */ - public long getNamedObject(String name) throws IOException { - checkIfClosed(); - - Map nameDirectory = getNameDirectory(); - Long recid = (Long) nameDirectory.get(name); - if (recid == null) { - return 0; - } - return recid.longValue(); - } - - /** - * Set the record id of a named object. - */ - public void setNamedObject(String name, long recid) throws IOException { - checkIfClosed(); - - Map nameDirectory = getNameDirectory(); - if (recid == 0) { - // remove from hashtable - nameDirectory.remove(name); - } else { - nameDirectory.put(name, new Long(recid)); - } - saveNameDirectory(nameDirectory); - } - - /** - * Commit (make persistent) all changes since beginning of transaction. - */ - public synchronized void commit() throws IOException { - checkIfClosed(); - - _pageman.commit(); - } - - /** - * Rollback (cancel) all changes since beginning of transaction. - */ - public synchronized void rollback() throws IOException { - checkIfClosed(); - - _pageman.rollback(); - } - - /** - * Load name directory - */ - private Map getNameDirectory() throws IOException { - // retrieve directory of named hashtable - long nameDirectory_recid = getRoot(NAME_DIRECTORY_ROOT); - if (nameDirectory_recid == 0) { - _nameDirectory = new HashMap(); - nameDirectory_recid = insert(_nameDirectory); - setRoot(NAME_DIRECTORY_ROOT, nameDirectory_recid); - } else { - _nameDirectory = (Map) fetch(nameDirectory_recid); - } - return _nameDirectory; - } - - private void saveNameDirectory(Map directory) throws IOException { - long recid = getRoot(NAME_DIRECTORY_ROOT); - if (recid == 0) { - throw new IOException("Name directory must exist"); - } - update(recid, _nameDirectory); - } - - /** - * Check if RecordManager has been closed. If so, throw an - * IllegalStateException. - */ - private void checkIfClosed() throws IllegalStateException { - if (_file == null) { - throw new IllegalStateException("RecordManager has been closed"); - } - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/DataPage.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/DataPage.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/DataPage.java (working copy) @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: DataPage.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * Class describing a page that holds data. - */ -final class DataPage extends PageHeader { - // offsets - private static final short O_FIRST = PageHeader.SIZE; // short firstrowid - static final short O_DATA = (short) (O_FIRST + Magic.SZ_SHORT); - static final short DATA_PER_PAGE = (short) (RecordFile.BLOCK_SIZE - O_DATA); - - /** - * Constructs a data page view from the indicated block. - */ - DataPage(BlockIo block) { - super(block); - } - - /** - * Factory method to create or return a data page for the indicated block. - */ - static DataPage getDataPageView(BlockIo block) { - BlockView view = block.getView(); - if (view != null && view instanceof DataPage) { - return (DataPage) view; - } else { - return new DataPage(block); - } - } - - /** Returns the first rowid's offset */ - short getFirst() { - return block.readShort(O_FIRST); - } - - /** Sets the first rowid's offset */ - void setFirst(short value) { - paranoiaMagicOk(); - if (value > 0 && value < O_DATA) { - throw new Error("DataPage.setFirst: offset " + value + " too small"); - } - block.writeShort(O_FIRST, value); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageManager.java (working copy) @@ -1,289 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: PageManager.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class manages the linked lists of pages that make up a file. - */ -final class PageManager { - // our record file - private RecordFile file; - // header data - private FileHeader header; - private BlockIo headerBuf; - - /** - * Creates a new page manager using the indicated record file. - */ - PageManager(RecordFile file) throws IOException { - this.file = file; - - // check the file header. If the magic is 0, we assume a new - // file. Note that we hold on to the file header node. - headerBuf = file.get(0); - if (headerBuf.readShort(0) == 0) { - header = new FileHeader(headerBuf, true); - } else { - header = new FileHeader(headerBuf, false); - } - } - - /** - * Allocates a page of the indicated type. Returns recid of the page. - */ - long allocate(short type) throws IOException { - - if (type == Magic.FREE_PAGE) { - throw new Error("allocate of free page?"); - } - - // do we have something on the free list? - long retval = header.getFirstOf(Magic.FREE_PAGE); - boolean isNew = false; - if (retval != 0) { - // yes. Point to it and make the next of that page the - // new first free page. - header.setFirstOf(Magic.FREE_PAGE, getNext(retval)); - } else { - // nope. make a new record - retval = header.getLastOf(Magic.FREE_PAGE); - if (retval == 0) { - // very new file - allocate record #1 - retval = 1; - } - header.setLastOf(Magic.FREE_PAGE, retval + 1); - isNew = true; - } - - // Cool. We have a record, add it to the correct list - BlockIo buf = file.get(retval); - PageHeader pageHdr = isNew ? new PageHeader(buf, type) : PageHeader - .getView(buf); - long oldLast = header.getLastOf(type); - - // Clean data. - System.arraycopy(RecordFile.cleanData, 0, buf.getData(), 0, - RecordFile.BLOCK_SIZE); - pageHdr.setType(type); - pageHdr.setPrev(oldLast); - pageHdr.setNext(0); - - if (oldLast == 0) { - // This was the first one of this type - header.setFirstOf(type, retval); - } - header.setLastOf(type, retval); - file.release(retval, true); - - // If there's a previous, fix up its pointer - if (oldLast != 0) { - buf = file.get(oldLast); - pageHdr = PageHeader.getView(buf); - pageHdr.setNext(retval); - file.release(oldLast, true); - } - - // remove the view, we have modified the type. - buf.setView(null); - - return retval; - } - - /** - * Frees a page of the indicated type. - */ - void free(short type, long recid) throws IOException { - if (type == Magic.FREE_PAGE) { - throw new Error("free free page?"); - } - if (recid == 0) { - throw new Error("free header page?"); - } - - // get the page and read next and previous pointers - BlockIo buf = file.get(recid); - PageHeader pageHdr = PageHeader.getView(buf); - long prev = pageHdr.getPrev(); - long next = pageHdr.getNext(); - - // put the page at the front of the free list. - pageHdr.setType(Magic.FREE_PAGE); - pageHdr.setNext(header.getFirstOf(Magic.FREE_PAGE)); - pageHdr.setPrev(0); - - header.setFirstOf(Magic.FREE_PAGE, recid); - file.release(recid, true); - - // remove the page from its old list - if (prev != 0) { - buf = file.get(prev); - pageHdr = PageHeader.getView(buf); - pageHdr.setNext(next); - file.release(prev, true); - } else { - header.setFirstOf(type, next); - } - if (next != 0) { - buf = file.get(next); - pageHdr = PageHeader.getView(buf); - pageHdr.setPrev(prev); - file.release(next, true); - } else { - header.setLastOf(type, prev); - } - - } - - /** - * Returns the page following the indicated block - */ - long getNext(long block) throws IOException { - try { - return PageHeader.getView(file.get(block)).getNext(); - } finally { - file.release(block, false); - } - } - - /** - * Returns the page before the indicated block - */ - long getPrev(long block) throws IOException { - try { - return PageHeader.getView(file.get(block)).getPrev(); - } finally { - file.release(block, false); - } - } - - /** - * Returns the first page on the indicated list. - */ - long getFirst(short type) throws IOException { - return header.getFirstOf(type); - } - - /** - * Returns the last page on the indicated list. - */ - long getLast(short type) throws IOException { - return header.getLastOf(type); - } - - /** - * Commit all pending (in-memory) data by flushing the page manager. This - * forces a flush of all outstanding blocks (this it's an implicit - * {@link RecordFile#commit} as well). - */ - void commit() throws IOException { - // write the header out - file.release(headerBuf); - file.commit(); - - // and obtain it again - headerBuf = file.get(0); - header = new FileHeader(headerBuf, false); - } - - /** - * Flushes the page manager. This forces a flush of all outstanding blocks - * (this it's an implicit {@link RecordFile#commit} as well). - */ - void rollback() throws IOException { - // release header - file.discard(headerBuf); - file.rollback(); - // and obtain it again - headerBuf = file.get(0); - if (headerBuf.readShort(0) == 0) { - header = new FileHeader(headerBuf, true); - } else { - header = new FileHeader(headerBuf, false); - } - } - - /** - * Closes the page manager. This flushes the page manager and releases the - * lock on the header. - */ - void close() throws IOException { - file.release(headerBuf); - file.commit(); - headerBuf = null; - header = null; - file = null; - } - - /** - * Returns the file header. - */ - FileHeader getFileHeader() { - return header; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Location.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Location.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Location.java (working copy) @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: Location.java,v 1.2 2003/11/01 14:17:21 dranatunga Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * This class represents a location within a file. Both physical and logical - * rowids are based on locations internally - this version is used when there is - * no file block to back the location's data. - */ -final class Location { - private final long block; - private final short offset; - - /** - * Creates a location from a (block, offset) tuple. - */ - Location(long block, short offset) { - this.block = block; - this.offset = offset; - } - - /** - * Creates a location from a combined block/offset long, as used in the - * external representation of logical rowids. - * - * @see #toLong() - */ - Location(long blockOffset) { - offset = (short) (blockOffset & 0xffff); - block = blockOffset >> 16; - } - - /** - * Creates a location based on the data of the physical rowid. - */ - Location(PhysicalRowId src) { - block = src.getBlock(); - offset = src.getOffset(); - } - - /** - * Returns the file block of the location - */ - long getBlock() { - return block; - } - - /** - * Returns the offset within the block of the location - */ - short getOffset() { - return offset; - } - - /** - * Returns the external representation of a location when used as a logical - * rowid, which combines the block and the offset in a single long. - */ - long toLong() { - return (block << 16) + offset; - } - - // overrides of java.lang.Object - - @Override - public boolean equals(Object o) { - if (o == null || !(o instanceof Location)) { - return false; - } - Location ol = (Location) o; - return ol.block == block && ol.offset == offset; - } - - @Override - public String toString() { - return "PL(" + block + ":" + offset + ")"; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPageManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPageManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPageManager.java (working copy) @@ -1,156 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FreeLogicalRowIdPageManager.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class manages free Logical rowid pages and provides methods to free and - * allocate Logical rowids on a high level. - */ -final class FreeLogicalRowIdPageManager { - // our record file - private final RecordFile file; - // our page manager - private final PageManager pageman; - - /** - * Creates a new instance using the indicated record file and page manager. - */ - FreeLogicalRowIdPageManager(RecordFile file, PageManager pageman) - throws IOException { - this.file = file; - this.pageman = pageman; - } - - /** - * Returns a free Logical rowid, or null if nothing was found. - */ - Location get() throws IOException { - - // Loop through the free Logical rowid list until we find - // the first rowid. - Location retval = null; - PageCursor curs = new PageCursor(pageman, Magic.FREELOGIDS_PAGE); - while (curs.next() != 0) { - FreeLogicalRowIdPage fp = FreeLogicalRowIdPage - .getFreeLogicalRowIdPageView(file.get(curs.getCurrent())); - int slot = fp.getFirstAllocated(); - if (slot != -1) { - // got one! - retval = new Location(fp.get(slot)); - fp.free(slot); - if (fp.getCount() == 0) { - // page became empty - free it - file.release(curs.getCurrent(), false); - pageman.free(Magic.FREELOGIDS_PAGE, curs.getCurrent()); - } else { - file.release(curs.getCurrent(), true); - } - - return retval; - } else { - // no luck, go to next page - file.release(curs.getCurrent(), false); - } - } - return null; - } - - /** - * Puts the indicated rowid on the free list - */ - void put(Location rowid) throws IOException { - - PhysicalRowId free = null; - PageCursor curs = new PageCursor(pageman, Magic.FREELOGIDS_PAGE); - long freePage = 0; - while (curs.next() != 0) { - freePage = curs.getCurrent(); - BlockIo curBlock = file.get(freePage); - FreeLogicalRowIdPage fp = FreeLogicalRowIdPage - .getFreeLogicalRowIdPageView(curBlock); - int slot = fp.getFirstFree(); - if (slot != -1) { - free = fp.alloc(slot); - break; - } - - file.release(curBlock); - } - if (free == null) { - // No more space on the free list, add a page. - freePage = pageman.allocate(Magic.FREELOGIDS_PAGE); - BlockIo curBlock = file.get(freePage); - FreeLogicalRowIdPage fp = FreeLogicalRowIdPage - .getFreeLogicalRowIdPageView(curBlock); - free = fp.alloc(0); - } - free.setBlock(rowid.getBlock()); - free.setOffset(rowid.getOffset()); - file.release(freePage, true); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/LogicalRowIdManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/LogicalRowIdManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/LogicalRowIdManager.java (working copy) @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: LogicalRowIdManager.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class manages the linked lists of logical rowid pages. - */ -final class LogicalRowIdManager { - // our record file and associated page manager - private final RecordFile file; - private final PageManager pageman; - private final FreeLogicalRowIdPageManager freeman; - - /** - * Creates a log rowid manager using the indicated record file and page - * manager - */ - LogicalRowIdManager(RecordFile file, PageManager pageman) throws IOException { - this.file = file; - this.pageman = pageman; - freeman = new FreeLogicalRowIdPageManager(file, pageman); - - } - - /** - * Creates a new logical rowid pointing to the indicated physical id - */ - Location insert(Location loc) throws IOException { - // check whether there's a free rowid to reuse - Location retval = freeman.get(); - if (retval == null) { - // no. This means that we bootstrap things by allocating - // a new translation page and freeing all the rowids on it. - long firstPage = pageman.allocate(Magic.TRANSLATION_PAGE); - short curOffset = TranslationPage.O_TRANS; - for (int i = 0; i < TranslationPage.ELEMS_PER_PAGE; i++) { - freeman.put(new Location(firstPage, curOffset)); - curOffset += PhysicalRowId.SIZE; - } - retval = freeman.get(); - if (retval == null) { - throw new Error("couldn't obtain free translation"); - } - } - // write the translation. - update(retval, loc); - return retval; - } - - /** - * Releases the indicated logical rowid. - */ - void delete(Location rowid) throws IOException { - - freeman.put(rowid); - } - - /** - * Updates the mapping - * - * @param rowid - * The logical rowid - * @param loc - * The physical rowid - */ - void update(Location rowid, Location loc) throws IOException { - - TranslationPage xlatPage = TranslationPage.getTranslationPageView(file - .get(rowid.getBlock())); - PhysicalRowId physid = xlatPage.get(rowid.getOffset()); - physid.setBlock(loc.getBlock()); - physid.setOffset(loc.getOffset()); - file.release(rowid.getBlock(), true); - } - - /** - * Returns a mapping - * - * @param rowid - * The logical rowid - * @return The physical rowid - */ - Location fetch(Location rowid) throws IOException { - - TranslationPage xlatPage = TranslationPage.getTranslationPageView(file - .get(rowid.getBlock())); - try { - Location retval = new Location(xlatPage.get(rowid.getOffset())); - return retval; - } finally { - file.release(rowid.getBlock(), false); - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageCursor.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageCursor.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageCursor.java (working copy) @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: PageCursor.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class provides a cursor that can follow lists of pages bi-directionally. - */ -final class PageCursor { - PageManager pageman; - long current; - short type; - - /** - * Constructs a page cursor that starts at the indicated block. - */ - PageCursor(PageManager pageman, long current) { - this.pageman = pageman; - this.current = current; - } - - /** - * Constructs a page cursor that starts at the first block of the indicated - * list. - */ - PageCursor(PageManager pageman, short type) throws IOException { - this.pageman = pageman; - this.type = type; - } - - /** - * Returns the current value of the cursor. - */ - long getCurrent() throws IOException { - return current; - } - - /** - * Returns the next value of the cursor - */ - long next() throws IOException { - if (current == 0) { - current = pageman.getFirst(type); - } else { - current = pageman.getNext(current); - } - return current; - } - - /** - * Returns the previous value of the cursor - */ - long prev() throws IOException { - current = pageman.getPrev(current); - return current; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPageManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPageManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPageManager.java (working copy) @@ -1,163 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FreePhysicalRowIdPageManager.java,v 1.2 2001/11/17 16:14:25 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class manages free physical rowid pages and provides methods to free and - * allocate physical rowids on a high level. - */ -final class FreePhysicalRowIdPageManager { - // our record file - protected RecordFile _file; - - // our page manager - protected PageManager _pageman; - - /** - * Creates a new instance using the indicated record file and page manager. - */ - FreePhysicalRowIdPageManager(RecordFile file, PageManager pageman) - throws IOException { - _file = file; - _pageman = pageman; - } - - /** - * Returns a free physical rowid of the indicated size, or null if nothing was - * found. - */ - Location get(int size) throws IOException { - // Loop through the free physical rowid list until we find - // a rowid that's large enough. - Location retval = null; - PageCursor curs = new PageCursor(_pageman, Magic.FREEPHYSIDS_PAGE); - - while (curs.next() != 0) { - FreePhysicalRowIdPage fp = FreePhysicalRowIdPage - .getFreePhysicalRowIdPageView(_file.get(curs.getCurrent())); - int slot = fp.getFirstLargerThan(size); - if (slot != -1) { - // got one! - retval = new Location(fp.get(slot)); - - fp.get(slot).getSize(); - fp.free(slot); - if (fp.getCount() == 0) { - // page became empty - free it - _file.release(curs.getCurrent(), false); - _pageman.free(Magic.FREEPHYSIDS_PAGE, curs.getCurrent()); - } else { - _file.release(curs.getCurrent(), true); - } - - return retval; - } else { - // no luck, go to next page - _file.release(curs.getCurrent(), false); - } - - } - return null; - } - - /** - * Puts the indicated rowid on the free list - */ - void put(Location rowid, int size) throws IOException { - - FreePhysicalRowId free = null; - PageCursor curs = new PageCursor(_pageman, Magic.FREEPHYSIDS_PAGE); - long freePage = 0; - while (curs.next() != 0) { - freePage = curs.getCurrent(); - BlockIo curBlock = _file.get(freePage); - FreePhysicalRowIdPage fp = FreePhysicalRowIdPage - .getFreePhysicalRowIdPageView(curBlock); - int slot = fp.getFirstFree(); - if (slot != -1) { - free = fp.alloc(slot); - break; - } - - _file.release(curBlock); - } - if (free == null) { - // No more space on the free list, add a page. - freePage = _pageman.allocate(Magic.FREEPHYSIDS_PAGE); - BlockIo curBlock = _file.get(freePage); - FreePhysicalRowIdPage fp = FreePhysicalRowIdPage - .getFreePhysicalRowIdPageView(curBlock); - free = fp.alloc(0); - } - - free.setBlock(rowid.getBlock()); - free.setOffset(rowid.getOffset()); - free.setSize(size); - _file.release(freePage, true); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowIdManager.java (working copy) @@ -1,334 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: PhysicalRowIdManager.java,v 1.3 2003/03/21 03:00:09 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class manages physical row ids, and their data. - */ -final class PhysicalRowIdManager { - - // The file we're talking to and the associated page manager. - private final RecordFile file; - private final PageManager pageman; - private final FreePhysicalRowIdPageManager freeman; - - /** - * Creates a new rowid manager using the indicated record file. and page - * manager. - */ - PhysicalRowIdManager(RecordFile file, PageManager pageManager) - throws IOException { - this.file = file; - pageman = pageManager; - freeman = new FreePhysicalRowIdPageManager(file, pageman); - } - - /** - * Inserts a new record. Returns the new physical rowid. - */ - Location insert(byte[] data, int start, int length) throws IOException { - Location retval = alloc(length); - write(retval, data, start, length); - return retval; - } - - /** - * Updates an existing record. Returns the possibly changed physical rowid. - */ - Location update(Location rowid, byte[] data, int start, int length) - throws IOException { - // fetch the record header - BlockIo block = file.get(rowid.getBlock()); - RecordHeader head = new RecordHeader(block, rowid.getOffset()); - if (length > head.getAvailableSize()) { - // not enough space - we need to copy to a new rowid. - file.release(block); - free(rowid); - rowid = alloc(length); - } else { - file.release(block); - } - - // 'nuff space, write it in and return the rowid. - write(rowid, data, start, length); - return rowid; - } - - /** - * Deletes a record. - */ - void delete(Location rowid) throws IOException { - free(rowid); - } - - /** - * Retrieves a record. - */ - byte[] fetch(Location rowid) throws IOException { - // fetch the record header - PageCursor curs = new PageCursor(pageman, rowid.getBlock()); - BlockIo block = file.get(curs.getCurrent()); - RecordHeader head = new RecordHeader(block, rowid.getOffset()); - - // allocate a return buffer - byte[] retval = new byte[head.getCurrentSize()]; - if (retval.length == 0) { - file.release(curs.getCurrent(), false); - return retval; - } - - // copy bytes in - int offsetInBuffer = 0; - int leftToRead = retval.length; - short dataOffset = (short) (rowid.getOffset() + RecordHeader.SIZE); - while (leftToRead > 0) { - // copy current page's data to return buffer - int toCopy = RecordFile.BLOCK_SIZE - dataOffset; - if (leftToRead < toCopy) { - toCopy = leftToRead; - } - System.arraycopy(block.getData(), dataOffset, retval, offsetInBuffer, - toCopy); - - // Go to the next block - leftToRead -= toCopy; - offsetInBuffer += toCopy; - - file.release(block); - - if (leftToRead > 0) { - block = file.get(curs.next()); - dataOffset = DataPage.O_DATA; - } - - } - - return retval; - } - - /** - * Allocate a new rowid with the indicated size. - */ - private Location alloc(int size) throws IOException { - Location retval = freeman.get(size); - if (retval == null) { - retval = allocNew(size, pageman.getLast(Magic.USED_PAGE)); - } - return retval; - } - - /** - * Allocates a new rowid. The second parameter is there to allow for a - * recursive call - it indicates where the search should start. - */ - private Location allocNew(int size, long start) throws IOException { - BlockIo curBlock; - DataPage curPage; - if (start == 0) { - // we need to create a new page. - start = pageman.allocate(Magic.USED_PAGE); - curBlock = file.get(start); - curPage = DataPage.getDataPageView(curBlock); - curPage.setFirst(DataPage.O_DATA); - RecordHeader hdr = new RecordHeader(curBlock, DataPage.O_DATA); - hdr.setAvailableSize(0); - hdr.setCurrentSize(0); - } else { - curBlock = file.get(start); - curPage = DataPage.getDataPageView(curBlock); - } - - // follow the rowids on this page to get to the last one. We don't - // fall off, because this is the last page, remember? - short pos = curPage.getFirst(); - if (pos == 0) { - // page is exactly filled by the last block of a record - file.release(curBlock); - return allocNew(size, 0); - } - - RecordHeader hdr = new RecordHeader(curBlock, pos); - while (hdr.getAvailableSize() != 0 && pos < RecordFile.BLOCK_SIZE) { - pos += hdr.getAvailableSize() + RecordHeader.SIZE; - if (pos == RecordFile.BLOCK_SIZE) { - // Again, a filled page. - file.release(curBlock); - return allocNew(size, 0); - } - - hdr = new RecordHeader(curBlock, pos); - } - - if (pos == RecordHeader.SIZE) { - // the last record exactly filled the page. Restart forcing - // a new page. - file.release(curBlock); - } - - // we have the position, now tack on extra pages until we've got - // enough space. - Location retval = new Location(start, pos); - int freeHere = RecordFile.BLOCK_SIZE - pos - RecordHeader.SIZE; - if (freeHere < size) { - // check whether the last page would have only a small bit left. - // if yes, increase the allocation. A small bit is a record - // header plus 16 bytes. - int lastSize = (size - freeHere) % DataPage.DATA_PER_PAGE; - if ((DataPage.DATA_PER_PAGE - lastSize) < (RecordHeader.SIZE + 16)) { - size += (DataPage.DATA_PER_PAGE - lastSize); - } - - // write out the header now so we don't have to come back. - hdr.setAvailableSize(size); - file.release(start, true); - - int neededLeft = size - freeHere; - // Refactor these two blocks! - while (neededLeft >= DataPage.DATA_PER_PAGE) { - start = pageman.allocate(Magic.USED_PAGE); - curBlock = file.get(start); - curPage = DataPage.getDataPageView(curBlock); - curPage.setFirst((short) 0); // no rowids, just data - file.release(start, true); - neededLeft -= DataPage.DATA_PER_PAGE; - } - if (neededLeft > 0) { - // done with whole chunks, allocate last fragment. - start = pageman.allocate(Magic.USED_PAGE); - curBlock = file.get(start); - curPage = DataPage.getDataPageView(curBlock); - curPage.setFirst((short) (DataPage.O_DATA + neededLeft)); - file.release(start, true); - } - } else { - // just update the current page. If there's less than 16 bytes - // left, we increase the allocation (16 bytes is an arbitrary - // number). - if (freeHere - size <= (16 + RecordHeader.SIZE)) { - size = freeHere; - } - hdr.setAvailableSize(size); - file.release(start, true); - } - return retval; - - } - - private void free(Location id) throws IOException { - // get the rowid, and write a zero current size into it. - BlockIo curBlock = file.get(id.getBlock()); - DataPage.getDataPageView(curBlock); - RecordHeader hdr = new RecordHeader(curBlock, id.getOffset()); - hdr.setCurrentSize(0); - file.release(id.getBlock(), true); - - // write the rowid to the free list - freeman.put(id, hdr.getAvailableSize()); - } - - /** - * Writes out data to a rowid. Assumes that any resizing has been done. - */ - private void write(Location rowid, byte[] data, int start, int length) - throws IOException { - PageCursor curs = new PageCursor(pageman, rowid.getBlock()); - BlockIo block = file.get(curs.getCurrent()); - RecordHeader hdr = new RecordHeader(block, rowid.getOffset()); - hdr.setCurrentSize(length); - if (length == 0) { - file.release(curs.getCurrent(), true); - return; - } - - // copy bytes in - int offsetInBuffer = start; - int leftToWrite = length; - short dataOffset = (short) (rowid.getOffset() + RecordHeader.SIZE); - while (leftToWrite > 0) { - // copy current page's data to return buffer - int toCopy = RecordFile.BLOCK_SIZE - dataOffset; - - if (leftToWrite < toCopy) { - toCopy = leftToWrite; - } - System.arraycopy(data, offsetInBuffer, block.getData(), dataOffset, - toCopy); - - // Go to the next block - leftToWrite -= toCopy; - offsetInBuffer += toCopy; - - file.release(curs.getCurrent(), true); - - if (leftToWrite > 0) { - block = file.get(curs.next()); - dataOffset = DataPage.O_DATA; - } - } - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordFile.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordFile.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordFile.java (working copy) @@ -1,453 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordFile.java,v 1.6 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; - -/** - * This class represents a random access file as a set of fixed size records. - * Each record has a physical record number, and records are cached in order to - * improve access. - *

- * The set of dirty records on the in-use list constitutes a transaction. Later - * on, we will send these records to some recovery thingy. - */ -public final class RecordFile { - final TransactionManager txnMgr; - - // Todo: reorganize in hashes and fifos as necessary. - // free -> inUse -> dirty -> inTxn -> free - // free is a cache, thus a FIFO. The rest are hashes. - private final LinkedList free = new LinkedList(); - private final HashMap inUse = new HashMap(); - private final HashMap dirty = new HashMap(); - private final HashMap inTxn = new HashMap(); - - // transactions disabled? - private boolean transactionsDisabled = false; - - /** The length of a single block. */ - public final static int BLOCK_SIZE = 8192;// 4096; - - /** The extension of a record file */ - final static String extension = ".db"; - - /** A block of clean data to wipe clean pages. */ - final static byte[] cleanData = new byte[BLOCK_SIZE]; - - private RandomAccessFile file; - private final String fileName; - - /** - * Creates a new object on the indicated filename. The file is opened in - * read/write mode. - * - * @param fileName - * the name of the file to open or create, without an extension. - * @throws IOException - * whenever the creation of the underlying RandomAccessFile throws - * it. - */ - RecordFile(String fileName) throws IOException { - this.fileName = fileName; - file = new RandomAccessFile(fileName + extension, "rw"); - txnMgr = new TransactionManager(this); - } - - /** - * Creates a new object on the indicated filename. The file is opened in - * read/write mode. - * - * @param fileName - * the name of the file to open or create, without an extension. - * @throws IOException - * whenever the creation of the underlying RandomAccessFile throws - * it. - */ - RecordFile(File file) throws IOException { - fileName = file.getName(); - this.file = new RandomAccessFile(file, "rw"); - txnMgr = new TransactionManager(this); - } - - /** - * Returns the file name. - */ - String getFileName() { - return fileName; - } - - /** - * Disables transactions: doesn't sync and doesn't use the transaction - * manager. - */ - void disableTransactions() { - transactionsDisabled = true; - } - - /** - * Gets a block from the file. The returned byte array is the in-memory copy - * of the record, and thus can be written (and subsequently released with a - * dirty flag in order to write the block back). - * - * @param blockid - * The record number to retrieve. - */ - BlockIo get(long blockid) throws IOException { - Long key = new Long(blockid); - - // try in transaction list, dirty list, free list - BlockIo node = (BlockIo) inTxn.get(key); - if (node != null) { - inTxn.remove(key); - inUse.put(key, node); - return node; - } - node = (BlockIo) dirty.get(key); - if (node != null) { - dirty.remove(key); - inUse.put(key, node); - return node; - } - for (Iterator i = free.iterator(); i.hasNext();) { - BlockIo cur = (BlockIo) i.next(); - if (cur.getBlockId() == blockid) { - node = cur; - i.remove(); - inUse.put(key, node); - return node; - } - } - - // sanity check: can't be on in use list - if (inUse.get(key) != null) { - throw new Error("double get for block " + blockid); - } - - // get a new node and read it from the file - node = getNewNode(blockid); - long offset = blockid * BLOCK_SIZE; - if (file.length() > 0 && offset <= file.length()) { - read(file, offset, node.getData(), BLOCK_SIZE); - } else { - System.arraycopy(cleanData, 0, node.getData(), 0, BLOCK_SIZE); - } - inUse.put(key, node); - node.setClean(); - return node; - } - - /** - * Releases a block. - * - * @param blockid - * The record number to release. - * @param isDirty - * If true, the block was modified since the get(). - */ - void release(long blockid, boolean isDirty) throws IOException { - BlockIo node = (BlockIo) inUse.get(new Long(blockid)); - if (node == null) { - throw new IOException("bad blockid " + blockid + " on release"); - } - if (!node.isDirty() && isDirty) { - node.setDirty(); - } - release(node); - } - - /** - * Releases a block. - * - * @param block - * The block to release. - */ - void release(BlockIo block) { - Long key = new Long(block.getBlockId()); - inUse.remove(key); - if (block.isDirty()) { - // System.out.println( "Dirty: " + key + block ); - dirty.put(key, block); - } else { - if (!transactionsDisabled && block.isInTransaction()) { - inTxn.put(key, block); - } else { - free.add(block); - } - } - } - - /** - * Discards a block (will not write the block even if it's dirty) - * - * @param block - * The block to discard. - */ - void discard(BlockIo block) { - Long key = new Long(block.getBlockId()); - inUse.remove(key); - - // note: block not added to free list on purpose, because - // it's considered invalid - } - - /** - * Commits the current transaction by flushing all dirty buffers to disk. - */ - void commit() throws IOException { - // debugging... - if (!inUse.isEmpty() && inUse.size() > 1) { - showList(inUse.values().iterator()); - throw new Error("in use list not empty at commit time (" + inUse.size() - + ")"); - } - - // System.out.println("committing..."); - - if (dirty.size() == 0) { - // if no dirty blocks, skip commit process - return; - } - - if (!transactionsDisabled) { - txnMgr.start(); - } - - for (Iterator i = dirty.values().iterator(); i.hasNext();) { - BlockIo node = (BlockIo) i.next(); - i.remove(); - // System.out.println("node " + node + " map size now " + dirty.size()); - if (transactionsDisabled) { - long offset = node.getBlockId() * BLOCK_SIZE; - file.seek(offset); - file.write(node.getData()); - node.setClean(); - free.add(node); - } else { - txnMgr.add(node); - inTxn.put(new Long(node.getBlockId()), node); - } - } - if (!transactionsDisabled) { - txnMgr.commit(); - } - } - - /** - * Rollback the current transaction by discarding all dirty buffers - */ - void rollback() throws IOException { - // debugging... - if (!inUse.isEmpty()) { - showList(inUse.values().iterator()); - throw new Error("in use list not empty at rollback time (" + inUse.size() - + ")"); - } - // System.out.println("rollback..."); - dirty.clear(); - - txnMgr.synchronizeLogFromDisk(); - - if (!inTxn.isEmpty()) { - showList(inTxn.values().iterator()); - throw new Error("in txn list not empty at rollback time (" + inTxn.size() - + ")"); - } - ; - } - - /** - * Commits and closes file. - */ - void close() throws IOException { - if (!dirty.isEmpty()) { - commit(); - } - txnMgr.shutdown(); - if (transactionsDisabled) { - txnMgr.removeLogFile(); - } - - if (!inTxn.isEmpty()) { - showList(inTxn.values().iterator()); - throw new Error("In transaction not empty"); - } - - // these actually ain't that bad in a production release - if (!dirty.isEmpty()) { - System.out.println("ERROR: dirty blocks at close time"); - showList(dirty.values().iterator()); - throw new Error("Dirty blocks at close time"); - } - if (!inUse.isEmpty()) { - System.out.println("ERROR: inUse blocks at close time"); - showList(inUse.values().iterator()); - throw new Error("inUse blocks at close time"); - } - - // debugging stuff to keep an eye on the free list - // System.out.println("Free list size:" + free.size()); - file.close(); - file = null; - } - - /** - * Force closing the file and underlying transaction manager. Used for testing - * purposed only. - */ - void forceClose() throws IOException { - txnMgr.forceClose(); - file.close(); - } - - /** - * Prints contents of a list - */ - private void showList(Iterator i) { - int cnt = 0; - while (i.hasNext()) { - System.out.println("elem " + cnt + ": " + i.next()); - cnt++; - } - } - - /** - * Returns a new node. The node is retrieved (and removed) from the released - * list or created new. - */ - private BlockIo getNewNode(long blockid) throws IOException { - - BlockIo retval = null; - if (!free.isEmpty()) { - retval = (BlockIo) free.removeFirst(); - } - if (retval == null) { - retval = new BlockIo(0, new byte[BLOCK_SIZE]); - } - - retval.setBlockId(blockid); - retval.setView(null); - return retval; - } - - /** - * Synchs a node to disk. This is called by the transaction manager's - * synchronization code. - */ - void synch(BlockIo node) throws IOException { - byte[] data = node.getData(); - if (data != null) { - long offset = node.getBlockId() * BLOCK_SIZE; - file.seek(offset); - file.write(data); - } - } - - /** - * Releases a node from the transaction list, if it was sitting there. - * - * @param recycle - * true if block data can be reused - */ - void releaseFromTransaction(BlockIo node, boolean recycle) throws IOException { - Long key = new Long(node.getBlockId()); - if ((inTxn.remove(key) != null) && recycle) { - free.add(node); - } - } - - /** - * Synchronizes the file. - */ - void sync() throws IOException { - file.getFD().sync(); - } - - /** - * Utility method: Read a block from a RandomAccessFile - */ - private static void read(RandomAccessFile file, long offset, byte[] buffer, - int nBytes) throws IOException { - file.seek(offset); - int remaining = nBytes; - int pos = 0; - while (remaining > 0) { - int read = file.read(buffer, pos, remaining); - if (read == -1) { - System.arraycopy(cleanData, 0, buffer, pos, remaining); - break; - } - remaining -= read; - pos += read; - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPage.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPage.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreeLogicalRowIdPage.java (working copy) @@ -1,173 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FreeLogicalRowIdPage.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * Class describing a page that holds logical rowids that were freed. Note that - * the methods have *physical* rowids in their signatures - this is because - * logical and physical rowids are internally the same, only their external - * representation (i.e. in the client API) differs. - */ -class FreeLogicalRowIdPage extends PageHeader { - // offsets - private static final short O_COUNT = PageHeader.SIZE; // short count - static final short O_FREE = (short) (O_COUNT + Magic.SZ_SHORT); - static final short ELEMS_PER_PAGE = (short) ((RecordFile.BLOCK_SIZE - O_FREE) / PhysicalRowId.SIZE); - - // slots we returned. - final PhysicalRowId[] slots = new PhysicalRowId[ELEMS_PER_PAGE]; - - /** - * Constructs a data page view from the indicated block. - */ - FreeLogicalRowIdPage(BlockIo block) { - super(block); - } - - /** - * Factory method to create or return a data page for the indicated block. - */ - static FreeLogicalRowIdPage getFreeLogicalRowIdPageView(BlockIo block) { - - BlockView view = block.getView(); - if (view != null && view instanceof FreeLogicalRowIdPage) { - return (FreeLogicalRowIdPage) view; - } else { - return new FreeLogicalRowIdPage(block); - } - } - - /** Returns the number of free rowids */ - short getCount() { - return block.readShort(O_COUNT); - } - - /** Sets the number of free rowids */ - private void setCount(short i) { - block.writeShort(O_COUNT, i); - } - - /** Frees a slot */ - void free(int slot) { - get(slot).setBlock(0); - setCount((short) (getCount() - 1)); - } - - /** Allocates a slot */ - PhysicalRowId alloc(int slot) { - setCount((short) (getCount() + 1)); - get(slot).setBlock(-1); - return get(slot); - } - - /** Returns true if a slot is allocated */ - boolean isAllocated(int slot) { - return get(slot).getBlock() > 0; - } - - /** Returns true if a slot is free */ - boolean isFree(int slot) { - return !isAllocated(slot); - } - - /** Returns the value of the indicated slot */ - PhysicalRowId get(int slot) { - if (slots[slot] == null) { - slots[slot] = new PhysicalRowId(block, slotToOffset(slot)); - } - ; - return slots[slot]; - } - - /** Converts slot to offset */ - private short slotToOffset(int slot) { - return (short) (O_FREE + (slot * PhysicalRowId.SIZE)); - } - - /** - * Returns first free slot, -1 if no slots are available - */ - int getFirstFree() { - for (int i = 0; i < ELEMS_PER_PAGE; i++) { - if (isFree(i)) { - return i; - } - } - return -1; - } - - /** - * Returns first allocated slot, -1 if no slots are available. - */ - int getFirstAllocated() { - for (int i = 0; i < ELEMS_PER_PAGE; i++) { - if (isAllocated(i)) { - return i; - } - } - return -1; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TransactionManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TransactionManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TransactionManager.java (working copy) @@ -1,435 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: TransactionManager.java,v 1.7 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Iterator; -import java.util.TreeSet; - -/** - * This class manages the transaction log that belongs to every - * {@link RecordFile}. The transaction log is either clean, or in progress. In - * the latter case, the transaction manager takes care of a roll forward. - *

- * Implementation note: this is a proof-of-concept implementation which hasn't - * been optimized for speed. For instance, all sorts of streams are created for - * every transaction. - */ -// TODO: Handle the case where we are recovering lg9 and lg0, were we -// should start with lg9 instead of lg0! - -public final class TransactionManager { - private final RecordFile owner; - - // streams for transaction log. - private FileOutputStream fos; - private ObjectOutputStream oos; - - /** - * By default, we keep 10 transactions in the log file before synchronizing it - * with the main database file. - */ - static final int DEFAULT_TXNS_IN_LOG = 10; - - /** - * Maximum number of transactions before the log file is synchronized with the - * main database file. - */ - private int _maxTxns = DEFAULT_TXNS_IN_LOG; - - /** - * In-core copy of transactions. We could read everything back from the log - * file, but the RecordFile needs to keep the dirty blocks in core anyway, so - * we might as well point to them and spare us a lot of hassle. - */ - private ArrayList[] txns = new ArrayList[DEFAULT_TXNS_IN_LOG]; - private int curTxn = -1; - - /** Extension of a log file. */ - static final String extension = ".lg"; - - /** log file name */ - private String logFileName; - - /** - * Instantiates a transaction manager instance. If recovery needs to be - * performed, it is done. - * - * @param owner - * the RecordFile instance that owns this transaction mgr. - */ - TransactionManager(RecordFile owner) throws IOException { - this.owner = owner; - logFileName = null; - recover(); - open(); - } - - /** - * Synchronize log file data with the main database file. - *

- * After this call, the main database file is guaranteed to be consistent and - * guaranteed to be the only file needed for backup purposes. - */ - public void synchronizeLog() throws IOException { - synchronizeLogFromMemory(); - } - - /** - * Set the maximum number of transactions to record in the log (and keep in - * memory) before the log is synchronized with the main database file. - *

- * This method must be called while there are no pending transactions in the - * log. - */ - public void setMaximumTransactionsInLog(int maxTxns) throws IOException { - if (maxTxns <= 0) { - throw new IllegalArgumentException( - "Argument 'maxTxns' must be greater than 0."); - } - if (curTxn != -1) { - throw new IllegalStateException( - "Cannot change setting while transactions are pending in the log"); - } - _maxTxns = maxTxns; - txns = new ArrayList[maxTxns]; - } - - /** Builds logfile name */ - private String makeLogName() { - return owner.getFileName() + extension; - } - - /** Synchs in-core transactions to data file and opens a fresh log */ - private void synchronizeLogFromMemory() throws IOException { - close(); - - TreeSet blockList = new TreeSet(new BlockIoComparator()); - - int numBlocks = 0; - int writtenBlocks = 0; - for (int i = 0; i < _maxTxns; i++) { - if (txns[i] == null) { - continue; - } - // Add each block to the blockList, replacing the old copy of this - // block if necessary, thus avoiding writing the same block twice - for (Iterator k = txns[i].iterator(); k.hasNext();) { - BlockIo block = (BlockIo) k.next(); - if (blockList.contains(block)) { - block.decrementTransactionCount(); - } else { - writtenBlocks++; - blockList.add(block); - } - numBlocks++; - } - - txns[i] = null; - } - // Write the blocks from the blockList to disk - synchronizeBlocks(blockList.iterator(), true); - - owner.sync(); - open(); - } - - /** Opens the log file */ - private void open() throws IOException { - logFileName = makeLogName(); - fos = new FileOutputStream(logFileName); - oos = new ObjectOutputStream(fos); - oos.writeShort(Magic.LOGFILE_HEADER); - oos.flush(); - curTxn = -1; - } - - /** Startup recovery on all files */ - private void recover() throws IOException { - String logName = makeLogName(); - File logFile = new File(logName); - if (!logFile.exists()) { - return; - } - if (logFile.length() == 0) { - logFile.delete(); - return; - } - - FileInputStream fis = new FileInputStream(logFile); - ObjectInputStream ois = new ObjectInputStream(fis); - - try { - if (ois.readShort() != Magic.LOGFILE_HEADER) { - throw new Error("Bad magic on log file"); - } - } catch (IOException e) { - // corrupted/empty logfile - logFile.delete(); - return; - } - - while (true) { - ArrayList blocks = null; - try { - blocks = (ArrayList) ois.readObject(); - } catch (ClassNotFoundException e) { - throw new Error("Unexcepted exception: " + e); - } catch (IOException e) { - // corrupted logfile, ignore rest of transactions - break; - } - synchronizeBlocks(blocks.iterator(), false); - - // ObjectInputStream must match exactly each - // ObjectOutputStream created during writes - try { - ois = new ObjectInputStream(fis); - } catch (IOException e) { - // corrupted logfile, ignore rest of transactions - break; - } - } - owner.sync(); - logFile.delete(); - } - - /** Synchronizes the indicated blocks with the owner. */ - private void synchronizeBlocks(Iterator blockIterator, boolean fromCore) - throws IOException { - // write block vector elements to the data file. - while (blockIterator.hasNext()) { - BlockIo cur = (BlockIo) blockIterator.next(); - owner.synch(cur); - if (fromCore) { - cur.decrementTransactionCount(); - if (!cur.isInTransaction()) { - owner.releaseFromTransaction(cur, true); - } - } - } - } - - /** Set clean flag on the blocks. */ - private void setClean(ArrayList blocks) throws IOException { - for (Iterator k = blocks.iterator(); k.hasNext();) { - BlockIo cur = (BlockIo) k.next(); - cur.setClean(); - } - } - - /** Discards the indicated blocks and notify the owner. */ - private void discardBlocks(ArrayList blocks) throws IOException { - for (Iterator k = blocks.iterator(); k.hasNext();) { - BlockIo cur = (BlockIo) k.next(); - cur.decrementTransactionCount(); - if (!cur.isInTransaction()) { - owner.releaseFromTransaction(cur, false); - } - } - } - - /** - * Starts a transaction. This can block if all slots have been filled with - * full transactions, waiting for the synchronization thread to clean out - * slots. - */ - void start() throws IOException { - curTxn++; - if (curTxn == _maxTxns) { - synchronizeLogFromMemory(); - curTxn = 0; - } - txns[curTxn] = new ArrayList(); - } - - /** - * Indicates the block is part of the transaction. - */ - void add(BlockIo block) throws IOException { - block.incrementTransactionCount(); - txns[curTxn].add(block); - } - - /** - * Commits the transaction to the log file. - */ - void commit() throws IOException { - oos.writeObject(txns[curTxn]); - sync(); - - // set clean flag to indicate blocks have been written to log - setClean(txns[curTxn]); - - // open a new ObjectOutputStream in order to store - // newer states of BlockIo - oos = new ObjectOutputStream(fos); - } - - /** Flushes and syncs */ - private void sync() throws IOException { - oos.flush(); - fos.flush(); - fos.getFD().sync(); - } - - /** - * Shutdowns the transaction manager. Resynchronizes outstanding logs. - */ - void shutdown() throws IOException { - synchronizeLogFromMemory(); - close(); - } - - /** - * Closes open files. - */ - private void close() throws IOException { - sync(); - oos.close(); - fos.close(); - oos = null; - fos = null; - } - - public void removeLogFile() { - // if file is not closed yet, just return - if (oos != null) { - return; - } - if (logFileName != null) { - File file = new File(logFileName); - file.delete(); - logFileName = null; - } - } - - /** - * Force closing the file without synchronizing pending transaction data. Used - * for testing purposes only. - */ - void forceClose() throws IOException { - oos.close(); - fos.close(); - oos = null; - fos = null; - } - - /** - * Use the disk-based transaction log to synchronize the data file. - * Outstanding memory logs are discarded because they are believed to be - * inconsistent. - */ - void synchronizeLogFromDisk() throws IOException { - close(); - - for (int i = 0; i < _maxTxns; i++) { - if (txns[i] == null) { - continue; - } - discardBlocks(txns[i]); - txns[i] = null; - } - - recover(); - open(); - } - - /** - * INNER CLASS. Comparator class for use by the tree set used to store the - * blocks to write for this transaction. The BlockIo objects are ordered by - * their blockIds. - */ - public static class BlockIoComparator implements Comparator { - - public int compare(Object o1, Object o2) { - BlockIo block1 = (BlockIo) o1; - BlockIo block2 = (BlockIo) o2; - int result = 0; - if (block1.getBlockId() == block2.getBlockId()) { - result = 0; - } else if (block1.getBlockId() < block2.getBlockId()) { - result = -1; - } else { - result = 1; - } - return result; - } - - @Override - public boolean equals(Object obj) { - return super.equals(obj); - } - } // class BlockIOComparator - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPage.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPage.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowIdPage.java (working copy) @@ -1,170 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FreePhysicalRowIdPage.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * Class describing a page that holds physical rowids that were freed. - */ -final class FreePhysicalRowIdPage extends PageHeader { - // offsets - private static final short O_COUNT = PageHeader.SIZE; // short count - static final short O_FREE = O_COUNT + Magic.SZ_SHORT; - static final short ELEMS_PER_PAGE = (RecordFile.BLOCK_SIZE - O_FREE) - / FreePhysicalRowId.SIZE; - - // slots we returned. - FreePhysicalRowId[] slots = new FreePhysicalRowId[ELEMS_PER_PAGE]; - - /** - * Constructs a data page view from the indicated block. - */ - FreePhysicalRowIdPage(BlockIo block) { - super(block); - } - - /** - * Factory method to create or return a data page for the indicated block. - */ - static FreePhysicalRowIdPage getFreePhysicalRowIdPageView(BlockIo block) { - BlockView view = block.getView(); - if (view != null && view instanceof FreePhysicalRowIdPage) { - return (FreePhysicalRowIdPage) view; - } else { - return new FreePhysicalRowIdPage(block); - } - } - - /** Returns the number of free rowids */ - short getCount() { - return block.readShort(O_COUNT); - } - - /** Sets the number of free rowids */ - private void setCount(short i) { - block.writeShort(O_COUNT, i); - } - - /** Frees a slot */ - void free(int slot) { - get(slot).setSize(0); - setCount((short) (getCount() - 1)); - } - - /** Allocates a slot */ - FreePhysicalRowId alloc(int slot) { - setCount((short) (getCount() + 1)); - return get(slot); - } - - /** Returns true if a slot is allocated */ - boolean isAllocated(int slot) { - return get(slot).getSize() != 0; - } - - /** Returns true if a slot is free */ - boolean isFree(int slot) { - return !isAllocated(slot); - } - - /** Returns the value of the indicated slot */ - FreePhysicalRowId get(int slot) { - if (slots[slot] == null) { - slots[slot] = new FreePhysicalRowId(block, slotToOffset(slot)); - } - ; - return slots[slot]; - } - - /** Converts slot to offset */ - short slotToOffset(int slot) { - return (short) (O_FREE + (slot * FreePhysicalRowId.SIZE)); - } - - /** - * Returns first free slot, -1 if no slots are available - */ - int getFirstFree() { - for (int i = 0; i < ELEMS_PER_PAGE; i++) { - if (isFree(i)) { - return i; - } - } - return -1; - } - - /** - * Returns first slot with available size >= indicated size, or -1 if no slots - * are available. - **/ - int getFirstLargerThan(int size) { - for (int i = 0; i < ELEMS_PER_PAGE; i++) { - if (isAllocated(i) && get(i).getSize() >= size) { - return i; - } - } - return -1; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Magic.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Magic.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Magic.java (working copy) @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: Magic.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * This interface contains magic cookies. - */ -public interface Magic { - /** Magic cookie at start of file */ - public short FILE_HEADER = 0x1350; - - /** Magic for blocks. They're offset by the block type magic codes. */ - public short BLOCK = 0x1351; - - /** Magics for blocks in certain lists. Offset by baseBlockMagic */ - short FREE_PAGE = 0; - short USED_PAGE = 1; - short TRANSLATION_PAGE = 2; - short FREELOGIDS_PAGE = 3; - short FREEPHYSIDS_PAGE = 4; - - /** Number of lists in a file */ - public short NLISTS = 5; - - /** - * Maximum number of blocks in a file, leaving room for a 16 bit offset - * encoded within a long. - */ - long MAX_BLOCKS = 0x7FFFFFFFFFFFL; - - /** Magic for transaction file */ - short LOGFILE_HEADER = 0x1360; - - /** Size of an externalized byte */ - public short SZ_BYTE = 1; - /** Size of an externalized short */ - public short SZ_SHORT = 2; - /** Size of an externalized int */ - public short SZ_INT = 4; - /** Size of an externalized long */ - public short SZ_LONG = 8; -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowId.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowId.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PhysicalRowId.java (working copy) @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: PhysicalRowId.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * A physical rowid is nothing else than a pointer to a physical location in a - * file - a (block, offset) tuple. - *

- * Note: The fact that the offset is modelled as a short limits the block - * size to 32k. - */ -class PhysicalRowId { - // offsets - private static final short O_BLOCK = 0; // long block - private static final short O_OFFSET = Magic.SZ_LONG; // short offset - static final int SIZE = O_OFFSET + Magic.SZ_SHORT; - - // my block and the position within the block - BlockIo block; - short pos; - - /** - * Constructs a physical rowid from the indicated data starting at the - * indicated position. - */ - PhysicalRowId(BlockIo block, short pos) { - this.block = block; - this.pos = pos; - } - - /** Returns the block number */ - long getBlock() { - return block.readLong(pos + O_BLOCK); - } - - /** Sets the block number */ - void setBlock(long value) { - block.writeLong(pos + O_BLOCK, value); - } - - /** Returns the offset */ - short getOffset() { - return block.readShort(pos + O_OFFSET); - } - - /** Sets the offset */ - void setOffset(short value) { - block.writeShort(pos + O_OFFSET, value); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockIo.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockIo.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockIo.java (working copy) @@ -1,313 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: BlockIo.java,v 1.2 2002/08/06 05:18:36 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; - -/** - * This class wraps a page-sized byte array and provides methods to read and - * write data to and from it. The readers and writers are just the ones that the - * rest of the toolkit needs, nothing else. Values written are compatible with - * java.io routines. - * - * @see java.io.DataInput - * @see java.io.DataOutput - */ -public final class BlockIo implements java.io.Externalizable { - - public final static long serialVersionUID = 2L; - - private long blockId; - - private transient byte[] data; // work area - private transient BlockView view = null; - private transient boolean dirty = false; - private transient int transactionCount = 0; - - /** - * Default constructor for serialization - */ - public BlockIo() { - // empty - } - - /** - * Constructs a new BlockIo instance working on the indicated buffer. - */ - BlockIo(long blockId, byte[] data) { - // removeme for production version - if (blockId > 10000000000L) { - throw new Error("bogus block id " + blockId); - } - this.blockId = blockId; - this.data = data; - } - - /** - * Returns the underlying array - */ - byte[] getData() { - return data; - } - - /** - * Sets the block number. Should only be called by RecordFile. - */ - void setBlockId(long id) { - if (isInTransaction()) { - throw new Error("BlockId assigned for transaction block"); - } - // removeme for production version - if (id > 10000000000L) { - throw new Error("bogus block id " + id); - } - blockId = id; - } - - /** - * Returns the block number. - */ - long getBlockId() { - return blockId; - } - - /** - * Returns the current view of the block. - */ - public BlockView getView() { - return view; - } - - /** - * Sets the current view of the block. - */ - public void setView(BlockView view) { - this.view = view; - } - - /** - * Sets the dirty flag - */ - void setDirty() { - dirty = true; - } - - /** - * Clears the dirty flag - */ - void setClean() { - dirty = false; - } - - /** - * Returns true if the dirty flag is set. - */ - boolean isDirty() { - return dirty; - } - - /** - * Returns true if the block is still dirty with respect to the transaction - * log. - */ - boolean isInTransaction() { - return transactionCount != 0; - } - - /** - * Increments transaction count for this block, to signal that this block is - * in the log but not yet in the data file. The method also takes a snapshot - * so that the data may be modified in new transactions. - */ - synchronized void incrementTransactionCount() { - transactionCount++; - // @fixme(alex) - setClean(); - } - - /** - * Decrements transaction count for this block, to signal that this block has - * been written from the log to the data file. - */ - synchronized void decrementTransactionCount() { - transactionCount--; - if (transactionCount < 0) { - throw new Error("transaction count on block " + getBlockId() - + " below zero!"); - } - - } - - /** - * Reads a byte from the indicated position - */ - public byte readByte(int pos) { - return data[pos]; - } - - /** - * Writes a byte to the indicated position - */ - public void writeByte(int pos, byte value) { - data[pos] = value; - setDirty(); - } - - /** - * Reads a short from the indicated position - */ - public short readShort(int pos) { - return (short) (((short) (data[pos + 0] & 0xff) << 8) | ((short) (data[pos + 1] & 0xff) << 0)); - } - - /** - * Writes a short to the indicated position - */ - public void writeShort(int pos, short value) { - data[pos + 0] = (byte) (0xff & (value >> 8)); - data[pos + 1] = (byte) (0xff & (value >> 0)); - setDirty(); - } - - /** - * Reads an int from the indicated position - */ - public int readInt(int pos) { - return (((data[pos + 0] & 0xff) << 24) | ((data[pos + 1] & 0xff) << 16) - | ((data[pos + 2] & 0xff) << 8) | ((data[pos + 3] & 0xff) << 0)); - } - - /** - * Writes an int to the indicated position - */ - public void writeInt(int pos, int value) { - data[pos + 0] = (byte) (0xff & (value >> 24)); - data[pos + 1] = (byte) (0xff & (value >> 16)); - data[pos + 2] = (byte) (0xff & (value >> 8)); - data[pos + 3] = (byte) (0xff & (value >> 0)); - setDirty(); - } - - /** - * Reads a long from the indicated position - */ - public long readLong(int pos) { - // Contributed by Erwin Bolwidt - // Gives about 15% performance improvement - return ((long) (((data[pos + 0] & 0xff) << 24) - | ((data[pos + 1] & 0xff) << 16) | ((data[pos + 2] & 0xff) << 8) | ((data[pos + 3] & 0xff))) << 32) - | ((long) (((data[pos + 4] & 0xff) << 24) - | ((data[pos + 5] & 0xff) << 16) | ((data[pos + 6] & 0xff) << 8) | ((data[pos + 7] & 0xff))) & 0xffffffff); - /* - * Original version by Alex Boisvert. Might be faster on 64-bit JVMs. return - * (((long)(data[pos+0] & 0xff) << 56) | ((long)(data[pos+1] & 0xff) << 48) - * | ((long)(data[pos+2] & 0xff) << 40) | ((long)(data[pos+3] & 0xff) << 32) - * | ((long)(data[pos+4] & 0xff) << 24) | ((long)(data[pos+5] & 0xff) << 16) - * | ((long)(data[pos+6] & 0xff) << 8) | ((long)(data[pos+7] & 0xff) << 0)); - */ - } - - /** - * Writes a long to the indicated position - */ - public void writeLong(int pos, long value) { - data[pos + 0] = (byte) (0xff & (value >> 56)); - data[pos + 1] = (byte) (0xff & (value >> 48)); - data[pos + 2] = (byte) (0xff & (value >> 40)); - data[pos + 3] = (byte) (0xff & (value >> 32)); - data[pos + 4] = (byte) (0xff & (value >> 24)); - data[pos + 5] = (byte) (0xff & (value >> 16)); - data[pos + 6] = (byte) (0xff & (value >> 8)); - data[pos + 7] = (byte) (0xff & (value >> 0)); - setDirty(); - } - - // overrides java.lang.Object - - @Override - public String toString() { - return "BlockIO(" + blockId + "," + dirty + "," + view + ")"; - } - - // implement externalizable interface - public void readExternal(ObjectInput in) throws IOException, - ClassNotFoundException { - blockId = in.readLong(); - int length = in.readInt(); - data = new byte[length]; - in.readFully(data); - } - - // implement externalizable interface - public void writeExternal(ObjectOutput out) throws IOException { - out.writeLong(blockId); - out.writeInt(data.length); - out.write(data); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowId.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowId.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FreePhysicalRowId.java (working copy) @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FreePhysicalRowId.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * This class extends the physical rowid with a size value to indicated the size - * of a free rowid on the free rowid list. - */ -final class FreePhysicalRowId extends PhysicalRowId { - // offsets - private static final short O_SIZE = PhysicalRowId.SIZE; // int size - static final short SIZE = O_SIZE + Magic.SZ_INT; - - /** - * Constructs a physical rowid from the indicated data starting at the - * indicated position. - */ - FreePhysicalRowId(BlockIo block, short pos) { - super(block, pos); - } - - /** Returns the size */ - int getSize() { - return block.readInt(pos + O_SIZE); - } - - /** Sets the size */ - void setSize(int value) { - block.writeInt(pos + O_SIZE, value); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockView.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockView.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/BlockView.java (working copy) @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: BlockView.java,v 1.2 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * This is a marker interface that is implemented by classes that interpret - * blocks of data by pretending to be an overlay. - * - * @see BlockIo#setView - */ -public interface BlockView { -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageHeader.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageHeader.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/PageHeader.java (working copy) @@ -1,177 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: PageHeader.java,v 1.2 2003/09/21 15:47:01 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class represents a page header. It is the common superclass for all - * different page views. - */ -public class PageHeader implements BlockView { - // offsets - private static final short O_MAGIC = 0; // short magic - private static final short O_NEXT = Magic.SZ_SHORT; // long next - private static final short O_PREV = O_NEXT + Magic.SZ_LONG; // long prev - protected static final short SIZE = O_PREV + Magic.SZ_LONG; - - // my block - protected BlockIo block; - - /** - * Constructs a PageHeader object from a block - * - * @param block - * The block that contains the file header - * @throws IOException - * if the block is too short to keep the file header. - */ - protected PageHeader(BlockIo block) { - initialize(block); - if (!magicOk()) { - throw new Error("CRITICAL: page header magic for block " - + block.getBlockId() + " not OK " + getMagic()); - } - } - - /** - * Constructs a new PageHeader of the indicated type. Used for newly created - * pages. - */ - PageHeader(BlockIo block, short type) { - initialize(block); - setType(type); - } - - /** - * Factory method to create or return a page header for the indicated block. - */ - static PageHeader getView(BlockIo block) { - BlockView view = block.getView(); - if (view != null && view instanceof PageHeader) { - return (PageHeader) view; - } else { - return new PageHeader(block); - } - } - - private void initialize(BlockIo block) { - this.block = block; - block.setView(this); - } - - /** - * Returns true if the magic corresponds with the fileHeader magic. - */ - private boolean magicOk() { - int magic = getMagic(); - return magic >= Magic.BLOCK - && magic <= (Magic.BLOCK + Magic.FREEPHYSIDS_PAGE); - } - - /** - * For paranoia mode - */ - protected void paranoiaMagicOk() { - if (!magicOk()) { - throw new Error("CRITICAL: page header magic not OK " + getMagic()); - } - } - - /** Returns the magic code */ - short getMagic() { - return block.readShort(O_MAGIC); - } - - /** Returns the next block. */ - long getNext() { - paranoiaMagicOk(); - return block.readLong(O_NEXT); - } - - /** Sets the next block. */ - void setNext(long next) { - paranoiaMagicOk(); - block.writeLong(O_NEXT, next); - } - - /** Returns the previous block. */ - long getPrev() { - paranoiaMagicOk(); - return block.readLong(O_PREV); - } - - /** Sets the previous block. */ - void setPrev(long prev) { - paranoiaMagicOk(); - block.writeLong(O_PREV, prev); - } - - /** Sets the type of the page header */ - void setType(short type) { - block.writeShort(O_MAGIC, (short) (Magic.BLOCK + type)); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordHeader.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordHeader.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/RecordHeader.java (working copy) @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordHeader.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * The data that comes at the start of a record of data. It stores both the - * current size and the avaliable size for the record - the latter can be bigger - * than the former, which allows the record to grow without needing to be moved - * and which allows the system to put small records in larger free spots. - */ -class RecordHeader { - // offsets - private static final short O_CURRENTSIZE = 0; // int currentSize - private static final short O_AVAILABLESIZE = Magic.SZ_INT; // int - // availableSize - static final int SIZE = O_AVAILABLESIZE + Magic.SZ_INT; - - // my block and the position within the block - private final BlockIo block; - private final short pos; - - /** - * Constructs a record header from the indicated data starting at the - * indicated position. - */ - RecordHeader(BlockIo block, short pos) { - this.block = block; - this.pos = pos; - if (pos > (RecordFile.BLOCK_SIZE - SIZE)) { - throw new Error("Offset too large for record header (" - + block.getBlockId() + ":" + pos + ")"); - } - } - - /** Returns the current size */ - int getCurrentSize() { - return block.readInt(pos + O_CURRENTSIZE); - } - - /** Sets the current size */ - void setCurrentSize(int value) { - block.writeInt(pos + O_CURRENTSIZE, value); - } - - /** Returns the available size */ - int getAvailableSize() { - return block.readInt(pos + O_AVAILABLESIZE); - } - - /** Sets the available size */ - void setAvailableSize(int value) { - block.writeInt(pos + O_AVAILABLESIZE, value); - } - - // overrides java.lang.Object - @Override - public String toString() { - return "RH(" + block.getBlockId() + ":" + pos + ", avl=" - + getAvailableSize() + ", cur=" + getCurrentSize() + ")"; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FileHeader.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FileHeader.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/FileHeader.java (working copy) @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: FileHeader.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; - -/** - * This class represents a file header. It is a 1:1 representation of the data - * that appears in block 0 of a file. - */ -class FileHeader implements BlockView { - // offsets - private static final short O_MAGIC = 0; // short magic - private static final short O_LISTS = Magic.SZ_SHORT; // long[2*NLISTS] - private static final int O_ROOTS = O_LISTS - + (Magic.NLISTS * 2 * Magic.SZ_LONG); - - // my block - private final BlockIo block; - - /** The number of "root" rowids available in the file. */ - static final int NROOTS = (RecordFile.BLOCK_SIZE - O_ROOTS) / Magic.SZ_LONG; - - /** - * Constructs a FileHeader object from a block. - * - * @param block - * The block that contains the file header - * @param isNew - * If true, the file header is for a new file. - * @throws IOException - * if the block is too short to keep the file header. - */ - FileHeader(BlockIo block, boolean isNew) { - this.block = block; - if (isNew) { - block.writeShort(O_MAGIC, Magic.FILE_HEADER); - } else if (!magicOk()) { - throw new Error("CRITICAL: file header magic not OK " - + block.readShort(O_MAGIC)); - } - } - - /** Returns true if the magic corresponds with the fileHeader magic. */ - private boolean magicOk() { - return block.readShort(O_MAGIC) == Magic.FILE_HEADER; - } - - /** Returns the offset of the "first" block of the indicated list */ - private short offsetOfFirst(int list) { - return (short) (O_LISTS + (2 * Magic.SZ_LONG * list)); - } - - /** Returns the offset of the "last" block of the indicated list */ - private short offsetOfLast(int list) { - return (short) (offsetOfFirst(list) + Magic.SZ_LONG); - } - - /** Returns the offset of the indicated root */ - private short offsetOfRoot(int root) { - return (short) (O_ROOTS + (root * Magic.SZ_LONG)); - } - - /** - * Returns the first block of the indicated list - */ - long getFirstOf(int list) { - return block.readLong(offsetOfFirst(list)); - } - - /** - * Sets the first block of the indicated list - */ - void setFirstOf(int list, long value) { - block.writeLong(offsetOfFirst(list), value); - } - - /** - * Returns the last block of the indicated list - */ - long getLastOf(int list) { - return block.readLong(offsetOfLast(list)); - } - - /** - * Sets the last block of the indicated list - */ - void setLastOf(int list, long value) { - block.writeLong(offsetOfLast(list), value); - } - - /** - * Returns the indicated root rowid. A root rowid is a special rowid that - * needs to be kept between sessions. It could conceivably be stored in a - * special file, but as a large amount of space in the block header is wasted - * anyway, it's more useful to store it where it belongs. - * - * @see #NROOTS - */ - long getRoot(int root) { - return block.readLong(offsetOfRoot(root)); - } - - /** - * Sets the indicated root rowid. - * - * @see #getRoot - * @see #NROOTS - */ - void setRoot(int root, long rowid) { - block.writeLong(offsetOfRoot(root), rowid); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Provider.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Provider.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/Provider.java (working copy) @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: Provider.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; - -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerOptions; -import org.apache.hadoop.hive.ql.util.jdbm.RecordManagerProvider; -import org.apache.hadoop.hive.ql.util.jdbm.helper.MRU; - -/** - * Provider of the default RecordManager implementation. - * - * @author Alex Boisvert - * @version $Id: Provider.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ -public final class Provider implements RecordManagerProvider { - - /** - * Create a default implementation record manager. - * - * @param name - * Name of the record file. - * @param options - * Record manager options. - * @throws IOException - * if an I/O related exception occurs while creating or opening the - * record manager. - * @throws UnsupportedOperationException - * if some options are not supported by the implementation. - * @throws IllegalArgumentException - * if some options are invalid. - */ - public RecordManager createRecordManager(String name, Properties options) - throws IOException { - RecordManager recman; - - recman = new BaseRecordManager(name); - recman = getCachedRecordManager(recman, options); - return recman; - } - - private RecordManager getCachedRecordManager(RecordManager recman, - Properties options) { - String value; - int cacheSize; - - value = options.getProperty(RecordManagerOptions.DISABLE_TRANSACTIONS, - "false"); - if (value.equalsIgnoreCase("TRUE")) { - ((BaseRecordManager) recman).disableTransactions(); - } - - value = options.getProperty(RecordManagerOptions.CACHE_SIZE, "1000"); - cacheSize = Integer.parseInt(value); - - value = options.getProperty(RecordManagerOptions.CACHE_TYPE, - RecordManagerOptions.NORMAL_CACHE); - if (value.equalsIgnoreCase(RecordManagerOptions.NORMAL_CACHE)) { - MRU cache = new MRU(cacheSize); - recman = new CacheRecordManager(recman, cache); - } else if (value.equalsIgnoreCase(RecordManagerOptions.SOFT_REF_CACHE)) { - throw new IllegalArgumentException("Soft reference cache not implemented"); - } else if (value.equalsIgnoreCase(RecordManagerOptions.WEAK_REF_CACHE)) { - throw new IllegalArgumentException("Weak reference cache not implemented"); - } else if (value.equalsIgnoreCase(RecordManagerOptions.NO_CACHE)) { - // do nothing - } else { - throw new IllegalArgumentException("Invalid cache type: " + value); - } - - return recman; - } - - public RecordManager createRecordManager(File file, Properties options) - throws IOException { - RecordManager recman = new BaseRecordManager(file); - recman = getCachedRecordManager(recman, options); - return recman; - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TranslationPage.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TranslationPage.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/TranslationPage.java (working copy) @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: TranslationPage.java,v 1.1 2000/05/06 00:00:31 boisvert Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -/** - * Class describing a page that holds translations from physical rowids to - * logical rowids. In fact, the page just holds physical rowids - the page's - * block is the block for the logical rowid, the offset serve as offset for the - * rowids. - */ -final class TranslationPage extends PageHeader { - // offsets - static final short O_TRANS = PageHeader.SIZE; // short count - static final short ELEMS_PER_PAGE = (RecordFile.BLOCK_SIZE - O_TRANS) - / PhysicalRowId.SIZE; - - // slots we returned. - final PhysicalRowId[] slots = new PhysicalRowId[ELEMS_PER_PAGE]; - - /** - * Constructs a data page view from the indicated block. - */ - TranslationPage(BlockIo block) { - super(block); - } - - /** - * Factory method to create or return a data page for the indicated block. - */ - static TranslationPage getTranslationPageView(BlockIo block) { - BlockView view = block.getView(); - if (view != null && view instanceof TranslationPage) { - return (TranslationPage) view; - } else { - return new TranslationPage(block); - } - } - - /** Returns the value of the indicated rowid on the page */ - PhysicalRowId get(short offset) { - int slot = (offset - O_TRANS) / PhysicalRowId.SIZE; - if (slots[slot] == null) { - slots[slot] = new PhysicalRowId(block, offset); - } - return slots[slot]; - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/CacheRecordManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/CacheRecordManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/recman/CacheRecordManager.java (working copy) @@ -1,436 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: CacheRecordManager.java,v 1.9 2005/06/25 23:12:32 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm.recman; - -import java.io.IOException; -import java.util.Enumeration; - -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.helper.CacheEvictionException; -import org.apache.hadoop.hive.ql.util.jdbm.helper.CachePolicy; -import org.apache.hadoop.hive.ql.util.jdbm.helper.CachePolicyListener; -import org.apache.hadoop.hive.ql.util.jdbm.helper.DefaultSerializer; -import org.apache.hadoop.hive.ql.util.jdbm.helper.Serializer; -import org.apache.hadoop.hive.ql.util.jdbm.helper.WrappedRuntimeException; - -/** - * A RecordManager wrapping and caching another RecordManager. - * - * @author Alex Boisvert - * @author Cees de Groot - * @version $Id: CacheRecordManager.java,v 1.9 2005/06/25 23:12:32 doomdark Exp - * $ - */ -public class CacheRecordManager implements RecordManager { - - /** - * Wrapped RecordManager - */ - protected RecordManager _recman; - - /** - * Cache for underlying RecordManager - */ - protected CachePolicy _cache; - - /** - * Construct a CacheRecordManager wrapping another RecordManager and using a - * given cache policy. - * - * @param recman - * Wrapped RecordManager - * @param cache - * Cache policy - */ - public CacheRecordManager(RecordManager recman, CachePolicy cache) { - if (recman == null) { - throw new IllegalArgumentException("Argument 'recman' is null"); - } - if (cache == null) { - throw new IllegalArgumentException("Argument 'cache' is null"); - } - _recman = recman; - _cache = cache; - - _cache.addListener(new CacheListener()); - } - - /** - * Get the underlying Record Manager. - * - * @return underlying RecordManager or null if CacheRecordManager has been - * closed. - */ - public RecordManager getRecordManager() { - return _recman; - } - - /** - * Get the underlying cache policy - * - * @return underlying CachePolicy or null if CacheRecordManager has been - * closed. - */ - public CachePolicy getCachePolicy() { - return _cache; - } - - /** - * Inserts a new record using a custom serializer. - * - * @param obj - * the object for the new record. - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public long insert(Object obj) throws IOException { - return insert(obj, DefaultSerializer.INSTANCE); - } - - /** - * Inserts a new record using a custom serializer. - * - * @param obj - * the object for the new record. - * @param serializer - * a custom serializer - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized long insert(Object obj, Serializer serializer) - throws IOException { - checkIfClosed(); - - long recid = _recman.insert(obj, serializer); - try { - _cache - .put(new Long(recid), new CacheEntry(recid, obj, serializer, false)); - } catch (CacheEvictionException except) { - throw new WrappedRuntimeException(except); - } - return recid; - } - - /** - * Deletes a record. - * - * @param recid - * the rowid for the record that should be deleted. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void delete(long recid) throws IOException { - checkIfClosed(); - - _recman.delete(recid); - _cache.remove(new Long(recid)); - } - - /** - * Updates a record using standard Java serialization. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public void update(long recid, Object obj) throws IOException { - update(recid, obj, DefaultSerializer.INSTANCE); - } - - /** - * Updates a record using a custom serializer. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @param serializer - * a custom serializer - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void update(long recid, Object obj, Serializer serializer) - throws IOException { - CacheEntry entry; - Long id; - - checkIfClosed(); - - id = new Long(recid); - try { - entry = (CacheEntry) _cache.get(id); - if (entry != null) { - // reuse existing cache entry - entry._obj = obj; - entry._serializer = serializer; - entry._isDirty = true; - } else { - _cache.put(id, new CacheEntry(recid, obj, serializer, true)); - } - } catch (CacheEvictionException except) { - throw new IOException(except.getMessage()); - } - } - - /** - * Fetches a record using standard Java serialization. - * - * @param recid - * the recid for the record that must be fetched. - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public Object fetch(long recid) throws IOException { - return fetch(recid, DefaultSerializer.INSTANCE); - } - - /** - * Fetches a record using a custom serializer. - * - * @param recid - * the recid for the record that must be fetched. - * @param serializer - * a custom serializer - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized Object fetch(long recid, Serializer serializer) - throws IOException { - checkIfClosed(); - - Long id = new Long(recid); - CacheEntry entry = (CacheEntry) _cache.get(id); - if (entry == null) { - entry = new CacheEntry(recid, null, serializer, false); - entry._obj = _recman.fetch(recid, serializer); - try { - _cache.put(id, entry); - } catch (CacheEvictionException except) { - throw new WrappedRuntimeException(except); - } - } - return entry._obj; - } - - /** - * Closes the record manager. - * - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public synchronized void close() throws IOException { - checkIfClosed(); - - updateCacheEntries(); - _recman.close(); - _recman = null; - _cache = null; - } - - /** - * Returns the number of slots available for "root" rowids. These slots can be - * used to store special rowids, like rowids that point to other rowids. Root - * rowids are useful for bootstrapping access to a set of data. - */ - public synchronized int getRootCount() { - checkIfClosed(); - - return _recman.getRootCount(); - } - - /** - * Returns the indicated root rowid. - * - * @see #getRootCount - */ - public synchronized long getRoot(int id) throws IOException { - checkIfClosed(); - - return _recman.getRoot(id); - } - - /** - * Sets the indicated root rowid. - * - * @see #getRootCount - */ - public synchronized void setRoot(int id, long rowid) throws IOException { - checkIfClosed(); - - _recman.setRoot(id, rowid); - } - - /** - * Commit (make persistent) all changes since beginning of transaction. - */ - public synchronized void commit() throws IOException { - checkIfClosed(); - updateCacheEntries(); - _recman.commit(); - } - - /** - * Rollback (cancel) all changes since beginning of transaction. - */ - public synchronized void rollback() throws IOException { - checkIfClosed(); - - _recman.rollback(); - - // discard all cache entries since we don't know which entries - // where part of the transaction - _cache.removeAll(); - } - - /** - * Obtain the record id of a named object. Returns 0 if named object doesn't - * exist. - */ - public synchronized long getNamedObject(String name) throws IOException { - checkIfClosed(); - - return _recman.getNamedObject(name); - } - - /** - * Set the record id of a named object. - */ - public synchronized void setNamedObject(String name, long recid) - throws IOException { - checkIfClosed(); - - _recman.setNamedObject(name, recid); - } - - /** - * Check if RecordManager has been closed. If so, throw an - * IllegalStateException - */ - private void checkIfClosed() throws IllegalStateException { - if (_recman == null) { - throw new IllegalStateException("RecordManager has been closed"); - } - } - - /** - * Update all dirty cache objects to the underlying RecordManager. - */ - protected void updateCacheEntries() throws IOException { - Enumeration enume = _cache.elements(); - while (enume.hasMoreElements()) { - CacheEntry entry = (CacheEntry) enume.nextElement(); - if (entry._isDirty) { - _recman.update(entry._recid, entry._obj, entry._serializer); - entry._isDirty = false; - } - } - } - - private class CacheEntry { - - long _recid; - Object _obj; - Serializer _serializer; - boolean _isDirty; - - CacheEntry(long recid, Object obj, Serializer serializer, boolean isDirty) { - _recid = recid; - _obj = obj; - _serializer = serializer; - _isDirty = isDirty; - } - - } // class CacheEntry - - private class CacheListener implements CachePolicyListener { - - /** - * Notification that cache is evicting an object - * - * @arg obj object evited from cache - * - */ - public void cacheObjectEvicted(Object obj) throws CacheEvictionException { - CacheEntry entry = (CacheEntry) obj; - if (entry._isDirty) { - try { - _recman.update(entry._recid, entry._obj, entry._serializer); - } catch (IOException except) { - throw new CacheEvictionException(except); - } - } - } - - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashBucket.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashBucket.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashBucket.java (working copy) @@ -1,300 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.htree; - -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.ArrayList; - -/** - * A bucket is a placeholder for multiple (key, value) pairs. Buckets are used - * to store collisions (same hash value) at all levels of an H*tree. - * - * There are two types of buckets: leaf and non-leaf. - * - * Non-leaf buckets are buckets which hold collisions which happen when the - * H*tree is not fully expanded. Keys in a non-leaf buckets can have different - * hash codes. Non-leaf buckets are limited to an arbitrary size. When this - * limit is reached, the H*tree should create a new Directory page and - * distribute keys of the non-leaf buckets into the newly created Directory. - * - * A leaf bucket is a bucket which contains keys which all have the same - * hashCode(). Leaf buckets stand at the bottom of an H*tree - * because the hashing algorithm cannot further discriminate between different - * keys based on their hash code. - * - * @author Alex Boisvert - * @version $Id: HashBucket.java,v 1.2 2005/06/25 23:12:32 doomdark Exp $ - */ -final class HashBucket extends HashNode implements Externalizable { - - final static long serialVersionUID = 1L; - - /** - * The maximum number of elements (key, value) a non-leaf bucket can contain. - */ - public static final int OVERFLOW_SIZE = 8; - - /** - * Depth of this bucket. - */ - private int _depth; - - /** - * Keys in this bucket. Keys are ordered to match their respective value in - * _values. - */ - private ArrayList _keys; - - /** - * Values in this bucket. Values are ordered to match their respective key in - * _keys. - */ - private ArrayList _values; - - /** - * Public constructor for serialization. - */ - public HashBucket() { - // empty - } - - /** - * Construct a bucket with a given depth level. Depth level is the number of - * HashDirectory above this bucket. - */ - public HashBucket(int level) { - if (level > HashDirectory.MAX_DEPTH + 1) { - throw new IllegalArgumentException( - "Cannot create bucket with depth > MAX_DEPTH+1. " + "Depth=" + level); - } - _depth = level; - _keys = new ArrayList(OVERFLOW_SIZE); - _values = new ArrayList(OVERFLOW_SIZE); - } - - /** - * Returns the number of elements contained in this bucket. - */ - public int getElementCount() { - return _keys.size(); - } - - /** - * Returns whether or not this bucket is a "leaf bucket". - */ - public boolean isLeaf() { - return (_depth > HashDirectory.MAX_DEPTH); - } - - /** - * Returns true if bucket can accept at least one more element. - */ - public boolean hasRoom() { - if (isLeaf()) { - return true; // leaf buckets are never full - } else { - // non-leaf bucket - return (_keys.size() < OVERFLOW_SIZE); - } - } - - /** - * Add an element (key, value) to this bucket. If an existing element has the - * same key, it is replaced silently. - * - * @return Object which was previously associated with the given key or - * null if no association existed. - */ - public Object addElement(Object key, Object value) { - int existing = _keys.indexOf(key); - if (existing != -1) { - // replace existing element - Object before = _values.get(existing); - _values.set(existing, value); - return before; - } else { - // add new (key, value) pair - _keys.add(key); - _values.add(value); - return null; - } - } - - /** - * Remove an element, given a specific key. - * - * @param key - * Key of the element to remove - * - * @return Removed element value, or null if not found - */ - public Object removeElement(Object key) { - int existing = _keys.indexOf(key); - if (existing != -1) { - Object obj = _values.get(existing); - _keys.remove(existing); - _values.remove(existing); - return obj; - } else { - // not found - return null; - } - } - - /** - * Returns the value associated with a given key. If the given key is not - * found in this bucket, returns null. - */ - public Object getValue(Object key) { - int existing = _keys.indexOf(key); - if (existing != -1) { - return _values.get(existing); - } else { - // key not found - return null; - } - } - - /** - * Obtain keys contained in this buckets. Keys are ordered to match their - * values, which be be obtained by calling getValues(). - * - * As an optimization, the Vector returned is the instance member of this - * class. Please don't modify outside the scope of this class. - */ - ArrayList getKeys() { - return _keys; - } - - /** - * Obtain values contained in this buckets. Values are ordered to match their - * keys, which be be obtained by calling getKeys(). - * - * As an optimization, the Vector returned is the instance member of this - * class. Please don't modify outside the scope of this class. - */ - ArrayList getValues() { - return _values; - } - - /** - * Implement Externalizable interface. - */ - public void writeExternal(ObjectOutput out) throws IOException { - out.writeInt(_depth); - - int entries = _keys.size(); - out.writeInt(entries); - - // write keys - for (int i = 0; i < entries; i++) { - out.writeObject(_keys.get(i)); - } - // write values - for (int i = 0; i < entries; i++) { - out.writeObject(_values.get(i)); - } - } - - /** - * Implement Externalizable interface. - */ - public void readExternal(ObjectInput in) throws IOException, - ClassNotFoundException { - _depth = in.readInt(); - - int entries = in.readInt(); - - // prepare array lists - int size = Math.max(entries, OVERFLOW_SIZE); - _keys = new ArrayList(size); - _values = new ArrayList(size); - - // read keys - for (int i = 0; i < entries; i++) { - _keys.add(in.readObject()); - } - // read values - for (int i = 0; i < entries; i++) { - _values.add(in.readObject()); - } - } - - @Override - public String toString() { - StringBuffer buf = new StringBuffer(); - buf.append("HashBucket {depth="); - buf.append(_depth); - buf.append(", keys="); - buf.append(_keys); - buf.append(", values="); - buf.append(_values); - buf.append("}"); - return buf.toString(); - } -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HTree.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HTree.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HTree.java (working copy) @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are (C) Copyright 2000 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.htree; - -import java.io.IOException; - -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.helper.FastIterator; - -/** - * Persistent hashtable implementation for PageManager. Implemented as an H*Tree - * structure. - * - * WARNING! If this instance is used in a transactional context, it *must* be - * discarded after a rollback. - * - * @author Alex Boisvert - * @version $Id: HTree.java,v 1.3 2005/06/25 23:12:32 doomdark Exp $ - */ -public class HTree { - - /** - * Root hash directory. - */ - private final HashDirectory _root; - - /** - * Private constructor - * - * @param root - * Root hash directory. - */ - private HTree(HashDirectory root) { - _root = root; - } - - /** - * Create a persistent hashtable. - * - * @param recman - * Record manager used for persistence. - */ - public static HTree createInstance(RecordManager recman) throws IOException { - HashDirectory root; - long recid; - - root = new HashDirectory((byte) 0); - recid = recman.insert(root); - root.setPersistenceContext(recman, recid); - - return new HTree(root); - } - - /** - * Load a persistent hashtable - * - * @param recman - * RecordManager used to store the persistent hashtable - * @param root_recid - * Record id of the root directory of the HTree - */ - public static HTree load(RecordManager recman, long root_recid) - throws IOException { - HTree tree; - HashDirectory root; - - root = (HashDirectory) recman.fetch(root_recid); - root.setPersistenceContext(recman, root_recid); - tree = new HTree(root); - return tree; - } - - /** - * Associates the specified value with the specified key. - * - * @param key - * key with which the specified value is to be assocated. - * @param value - * value to be associated with the specified key. - */ - public synchronized void put(Object key, Object value) throws IOException { - _root.put(key, value); - } - - /** - * Returns the value which is associated with the given key. Returns - * null if there is not association for this key. - * - * @param key - * key whose associated value is to be returned - */ - public synchronized Object get(Object key) throws IOException { - return _root.get(key); - } - - /** - * Remove the value which is associated with the given key. If the key does - * not exist, this method simply ignores the operation. - * - * @param key - * key whose associated value is to be removed - */ - public synchronized void remove(Object key) throws IOException { - _root.remove(key); - } - - /** - * Returns an enumeration of the keys contained in this - */ - public synchronized FastIterator keys() throws IOException { - return _root.keys(); - } - - /** - * Returns an enumeration of the values contained in this - */ - public synchronized FastIterator values() throws IOException { - return _root.values(); - } - - /** - * Get the record identifier used to load this hashtable. - */ - public long getRecid() { - return _root.getRecid(); - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashDirectory.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashDirectory.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashDirectory.java (working copy) @@ -1,535 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.htree; - -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; -import java.util.ArrayList; -import java.util.Iterator; - -import org.apache.hadoop.hive.ql.util.jdbm.RecordManager; -import org.apache.hadoop.hive.ql.util.jdbm.helper.FastIterator; -import org.apache.hadoop.hive.ql.util.jdbm.helper.IterationException; - -/** - * Hashtable directory page. - * - * @author Alex Boisvert - * @version $Id: HashDirectory.java,v 1.5 2005/06/25 23:12:32 doomdark Exp $ - */ -final class HashDirectory extends HashNode implements Externalizable { - - static final long serialVersionUID = 1L; - - /** - * Maximum number of children in a directory. - * - * (Must be a power of 2 -- if you update this value, you must also update - * BIT_SIZE and MAX_DEPTH.) - */ - static final int MAX_CHILDREN = 256; - - /** - * Number of significant bits per directory level. - */ - static final int BIT_SIZE = 8; // log2(256) = 8 - - /** - * Maximum number of levels (zero-based) - * - * (4 * 8 bits = 32 bits, which is the size of an "int", and as you know, - * hashcodes in Java are "ints") - */ - static final int MAX_DEPTH = 3; // 4 levels - - /** - * Record ids of children pages. - */ - private long[] _children; - - /** - * Depth of this directory page, zero-based - */ - private byte _depth; - - /** - * PageManager used to persist changes in directory and buckets - */ - private transient RecordManager _recman; - - /** - * This directory's record ID in the PageManager. (transient) - */ - private transient long _recid; - - /** - * Public constructor used by serialization - */ - public HashDirectory() { - // empty - } - - /** - * Construct a HashDirectory - * - * @param depth - * Depth of this directory page. - */ - HashDirectory(byte depth) { - _depth = depth; - _children = new long[MAX_CHILDREN]; - } - - /** - * Sets persistence context. This method must be called before any - * persistence-related operation. - * - * @param recman - * RecordManager which stores this directory - * @param recid - * Record id of this directory. - */ - void setPersistenceContext(RecordManager recman, long recid) { - _recman = recman; - _recid = recid; - } - - /** - * Get the record identifier used to load this hashtable. - */ - long getRecid() { - return _recid; - } - - /** - * Returns whether or not this directory is empty. A directory is empty when - * it no longer contains buckets or sub-directories. - */ - boolean isEmpty() { - for (long element : _children) { - if (element != 0) { - return false; - } - } - return true; - } - - /** - * Returns the value which is associated with the given key. Returns - * null if there is not association for this key. - * - * @param key - * key whose associated value is to be returned - */ - Object get(Object key) throws IOException { - int hash = hashCode(key); - long child_recid = _children[hash]; - if (child_recid == 0) { - // not bucket/page --> not found - return null; - } else { - HashNode node = (HashNode) _recman.fetch(child_recid); - // System.out.println("HashDirectory.get() child is : "+node); - - if (node instanceof HashDirectory) { - // recurse into next directory level - HashDirectory dir = (HashDirectory) node; - dir.setPersistenceContext(_recman, child_recid); - return dir.get(key); - } else { - // node is a bucket - HashBucket bucket = (HashBucket) node; - return bucket.getValue(key); - } - } - } - - /** - * Associates the specified value with the specified key. - * - * @param key - * key with which the specified value is to be assocated. - * @param value - * value to be associated with the specified key. - * @return object which was previously associated with the given key, or - * null if no association existed. - */ - Object put(Object key, Object value) throws IOException { - if (value == null) { - return remove(key); - } - int hash = hashCode(key); - long child_recid = _children[hash]; - if (child_recid == 0) { - // no bucket/page here yet, let's create a bucket - HashBucket bucket = new HashBucket(_depth + 1); - - // insert (key,value) pair in bucket - Object existing = bucket.addElement(key, value); - - long b_recid = _recman.insert(bucket); - _children[hash] = b_recid; - - _recman.update(_recid, this); - - // System.out.println("Added: "+bucket); - return existing; - } else { - HashNode node = (HashNode) _recman.fetch(child_recid); - - if (node instanceof HashDirectory) { - // recursive insert in next directory level - HashDirectory dir = (HashDirectory) node; - dir.setPersistenceContext(_recman, child_recid); - return dir.put(key, value); - } else { - // node is a bucket - HashBucket bucket = (HashBucket) node; - if (bucket.hasRoom()) { - Object existing = bucket.addElement(key, value); - _recman.update(child_recid, bucket); - // System.out.println("Added: "+bucket); - return existing; - } else { - // overflow, so create a new directory - if (_depth == MAX_DEPTH) { - throw new RuntimeException("Cannot create deeper directory. " - + "Depth=" + _depth); - } - HashDirectory dir = new HashDirectory((byte) (_depth + 1)); - long dir_recid = _recman.insert(dir); - dir.setPersistenceContext(_recman, dir_recid); - - _children[hash] = dir_recid; - _recman.update(_recid, this); - - // discard overflown bucket - _recman.delete(child_recid); - - // migrate existing bucket elements - ArrayList keys = bucket.getKeys(); - ArrayList values = bucket.getValues(); - int entries = keys.size(); - for (int i = 0; i < entries; i++) { - dir.put(keys.get(i), values.get(i)); - } - - // (finally!) insert new element - return dir.put(key, value); - } - } - } - } - - /** - * Remove the value which is associated with the given key. If the key does - * not exist, this method simply ignores the operation. - * - * @param key - * key whose associated value is to be removed - * @return object which was associated with the given key, or - * null if no association existed with given key. - */ - Object remove(Object key) throws IOException { - int hash = hashCode(key); - long child_recid = _children[hash]; - if (child_recid == 0) { - // not bucket/page --> not found - return null; - } else { - HashNode node = (HashNode) _recman.fetch(child_recid); - // System.out.println("HashDirectory.remove() child is : "+node); - - if (node instanceof HashDirectory) { - // recurse into next directory level - HashDirectory dir = (HashDirectory) node; - dir.setPersistenceContext(_recman, child_recid); - Object existing = dir.remove(key); - if (existing != null) { - if (dir.isEmpty()) { - // delete empty directory - _recman.delete(child_recid); - _children[hash] = 0; - _recman.update(_recid, this); - } - } - return existing; - } else { - // node is a bucket - HashBucket bucket = (HashBucket) node; - Object existing = bucket.removeElement(key); - if (existing != null) { - if (bucket.getElementCount() >= 1) { - _recman.update(child_recid, bucket); - } else { - // delete bucket, it's empty - _recman.delete(child_recid); - _children[hash] = 0; - _recman.update(_recid, this); - } - } - return existing; - } - } - } - - /** - * Calculates the hashcode of a key, based on the current directory depth. - */ - private int hashCode(Object key) { - int hashMask = hashMask(); - int hash = key.hashCode(); - hash = hash & hashMask; - hash = hash >>> ((MAX_DEPTH - _depth) * BIT_SIZE); - hash = hash % MAX_CHILDREN; - /* - * System.out.println("HashDirectory.hashCode() is: 0x" - * +Integer.toHexString(hash) +" for object hashCode() 0x" - * +Integer.toHexString(key.hashCode())); - */ - return hash; - } - - /** - * Calculates the hashmask of this directory. The hashmask is the bit mask - * applied to a hashcode to retain only bits that are relevant to this - * directory level. - */ - int hashMask() { - int bits = MAX_CHILDREN - 1; - int hashMask = bits << ((MAX_DEPTH - _depth) * BIT_SIZE); - /* - * System.out.println("HashDirectory.hashMask() is: 0x" - * +Integer.toHexString(hashMask)); - */ - return hashMask; - } - - /** - * Returns an enumeration of the keys contained in this - */ - FastIterator keys() throws IOException { - return new HDIterator(true); - } - - /** - * Returns an enumeration of the values contained in this - */ - FastIterator values() throws IOException { - return new HDIterator(false); - } - - /** - * Implement Externalizable interface - */ - public void writeExternal(ObjectOutput out) throws IOException { - out.writeByte(_depth); - out.writeObject(_children); - } - - /** - * Implement Externalizable interface - */ - public synchronized void readExternal(ObjectInput in) throws IOException, - ClassNotFoundException { - _depth = in.readByte(); - _children = (long[]) in.readObject(); - } - - // ////////////////////////////////////////////////////////////////////// - // INNER CLASS - // ////////////////////////////////////////////////////////////////////// - - /** - * Utility class to enumerate keys/values in a HTree - */ - public class HDIterator extends FastIterator { - - /** - * True if we're iterating on keys, False if enumerating on values. - */ - private final boolean _iterateKeys; - - /** - * Stacks of directories & last enumerated child position - */ - private final ArrayList _dirStack; - private final ArrayList _childStack; - - /** - * Current HashDirectory in the hierarchy - */ - private HashDirectory _dir; - - /** - * Current child position - */ - private int _child; - - /** - * Current bucket iterator - */ - private Iterator _iter; - - /** - * Construct an iterator on this directory. - * - * @param iterateKeys - * True if iteration supplies keys, False if iterateKeys supplies - * values. - */ - HDIterator(boolean iterateKeys) throws IOException { - _dirStack = new ArrayList(); - _childStack = new ArrayList(); - _dir = HashDirectory.this; - _child = -1; - _iterateKeys = iterateKeys; - - prepareNext(); - } - - /** - * Returns the next object. - */ - @Override - public Object next() { - Object next = null; - if (_iter != null && _iter.hasNext()) { - next = _iter.next(); - } else { - try { - prepareNext(); - } catch (IOException except) { - throw new IterationException(except); - } - if (_iter != null && _iter.hasNext()) { - return next(); - } - } - return next; - } - - /** - * Prepare internal state so we can answer hasMoreElements - * - * Actually, this code prepares an Enumeration on the next Bucket to - * enumerate. If no following bucket is found, the next Enumeration is set - * to null. - */ - private void prepareNext() throws IOException { - long child_recid = 0; - - // find next bucket/directory to enumerate - do { - _child++; - if (_child >= MAX_CHILDREN) { - - if (_dirStack.isEmpty()) { - // no more directory in the stack, we're finished - return; - } - - // try next page - _dir = (HashDirectory) _dirStack.remove(_dirStack.size() - 1); - _child = ((Integer) _childStack.remove(_childStack.size() - 1)) - .intValue(); - continue; - } - child_recid = _dir._children[_child]; - } while (child_recid == 0); - - if (child_recid == 0) { - throw new Error("child_recid cannot be 0"); - } - - HashNode node = (HashNode) _recman.fetch(child_recid); - // System.out.println("HDEnumeration.get() child is : "+node); - - if (node instanceof HashDirectory) { - // save current position - _dirStack.add(_dir); - _childStack.add(new Integer(_child)); - - _dir = (HashDirectory) node; - _child = -1; - - // recurse into - _dir.setPersistenceContext(_recman, child_recid); - prepareNext(); - } else { - // node is a bucket - HashBucket bucket = (HashBucket) node; - if (_iterateKeys) { - _iter = bucket.getKeys().iterator(); - } else { - _iter = bucket.getValues().iterator(); - } - } - } - } - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashNode.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashNode.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/htree/HashNode.java (working copy) @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - */ - -package org.apache.hadoop.hive.ql.util.jdbm.htree; - -import java.io.Serializable; - -/** - * Abstract class for Hashtable directory nodes - * - * @author Alex Boisvert - * @version $Id: HashNode.java,v 1.2 2003/03/21 02:54:58 boisvert Exp $ - */ -class HashNode implements Serializable { - - // Empty, there's no common functionality. We use this abstract - // class for typing only. - -} Index: src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManager.java =================================================================== --- src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManager.java (revision 1032800) +++ src/java/org/apache/hadoop/hive/ql/util/jdbm/RecordManager.java (working copy) @@ -1,231 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * JDBM LICENSE v1.00 - * - * Redistribution and use of this software and associated documentation - * ("Software"), with or without modification, are permitted provided - * that the following conditions are met: - * - * 1. Redistributions of source code must retain copyright - * statements and notices. Redistributions must also contain a - * copy of this document. - * - * 2. Redistributions in binary form must reproduce the - * above copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. - * - * 3. The name "JDBM" must not be used to endorse or promote - * products derived from this Software without prior written - * permission of Cees de Groot. For written permission, - * please contact cg@cdegroot.com. - * - * 4. Products derived from this Software may not be called "JDBM" - * nor may "JDBM" appear in their names without prior written - * permission of Cees de Groot. - * - * 5. Due credit should be given to the JDBM Project - * (http://jdbm.sourceforge.net/). - * - * THIS SOFTWARE IS PROVIDED BY THE JDBM PROJECT AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT - * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * CEES DE GROOT OR ANY CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * - * Copyright 2000 (C) Cees de Groot. All Rights Reserved. - * Copyright 2000-2001 (C) Alex Boisvert. All Rights Reserved. - * Contributions are Copyright (C) 2000 by their associated contributors. - * - * $Id: RecordManager.java,v 1.3 2005/06/25 23:12:31 doomdark Exp $ - */ - -package org.apache.hadoop.hive.ql.util.jdbm; - -import java.io.IOException; - -import org.apache.hadoop.hive.ql.util.jdbm.helper.Serializer; - -/** - * An interface to manages records, which are uninterpreted blobs of data. - *

- * The set of record operations is simple: fetch, insert, update and delete. - * Each record is identified using a "rowid" and contains a byte[] data block. - * Rowids are returned on inserts and you can store them someplace safe to be - * able to get back to them. Data blocks can be as long as you wish, and may - * have lengths different from the original when updating. - * - * @author Alex Boisvert - * @author Cees de Groot - * @version $Id: RecordManager.java,v 1.3 2005/06/25 23:12:31 doomdark Exp $ - */ -public interface RecordManager { - - /** - * Reserved slot for name directory. - */ - public static final int NAME_DIRECTORY_ROOT = 0; - - /** - * Inserts a new record using standard java object serialization. - * - * @param obj - * the object for the new record. - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract long insert(Object obj) throws IOException; - - /** - * Inserts a new record using a custom serializer. - * - * @param obj - * the object for the new record. - * @param serializer - * a custom serializer - * @return the rowid for the new record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract long insert(Object obj, Serializer serializer) - throws IOException; - - /** - * Deletes a record. - * - * @param recid - * the rowid for the record that should be deleted. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract void delete(long recid) throws IOException; - - /** - * Updates a record using standard java object serialization. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract void update(long recid, Object obj) throws IOException; - - /** - * Updates a record using a custom serializer. - * - * @param recid - * the recid for the record that is to be updated. - * @param obj - * the new object for the record. - * @param serializer - * a custom serializer - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract void update(long recid, Object obj, Serializer serializer) - throws IOException; - - /** - * Fetches a record using standard java object serialization. - * - * @param recid - * the recid for the record that must be fetched. - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract Object fetch(long recid) throws IOException; - - /** - * Fetches a record using a custom serializer. - * - * @param recid - * the recid for the record that must be fetched. - * @param serializer - * a custom serializer - * @return the object contained in the record. - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract Object fetch(long recid, Serializer serializer) - throws IOException; - - /** - * Closes the record manager. - * - * @throws IOException - * when one of the underlying I/O operations fails. - */ - public abstract void close() throws IOException; - - /** - * Returns the number of slots available for "root" rowids. These slots can be - * used to store special rowids, like rowids that point to other rowids. Root - * rowids are useful for bootstrapping access to a set of data. - */ - public abstract int getRootCount(); - - /** - * Returns the indicated root rowid. - * - * @see #getRootCount - */ - public abstract long getRoot(int id) throws IOException; - - /** - * Sets the indicated root rowid. - * - * @see #getRootCount - */ - public abstract void setRoot(int id, long rowid) throws IOException; - - /** - * Commit (make persistent) all changes since beginning of transaction. - */ - public abstract void commit() throws IOException; - - /** - * Rollback (cancel) all changes since beginning of transaction. - */ - public abstract void rollback() throws IOException; - - /** - * Obtain the record id of a named object. Returns 0 if named object doesn't - * exist. - */ - public abstract long getNamedObject(String name) throws IOException; - - /** - * Set the record id of a named object. - */ - public abstract void setNamedObject(String name, long recid) - throws IOException; - -} Index: src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java =================================================================== --- src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java (revision 1031087) +++ src/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java (working copy) @@ -6,13 +6,11 @@ package org.apache.hadoop.hive.ql.plan.api; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; -import org.apache.thrift.IntRangeSet; -import java.util.Map; import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.IntRangeSet; + public class OperatorType { public static final int JOIN = 0; public static final int MAPJOIN = 1; @@ -30,8 +28,8 @@ public static final int UDTF = 13; public static final int LATERALVIEWJOIN = 14; public static final int LATERALVIEWFORWARD = 15; - public static final int JDBMDUMMY = 16; - public static final int JDBMSINK = 17; + public static final int HASHTABLEDUMMY = 16; + public static final int HASHTABLESINK = 17; public static final IntRangeSet VALID_VALUES = new IntRangeSet( JOIN, @@ -50,8 +48,8 @@ UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, - JDBMDUMMY, - JDBMSINK); + HASHTABLEDUMMY, + HASHTABLESINK); public static final Map VALUES_TO_NAMES = new HashMap() {{ put(JOIN, "JOIN"); @@ -70,7 +68,7 @@ put(UDTF, "UDTF"); put(LATERALVIEWJOIN, "LATERALVIEWJOIN"); put(LATERALVIEWFORWARD, "LATERALVIEWFORWARD"); - put(JDBMDUMMY, "JDBMDUMMY"); - put(JDBMSINK, "JDBMSINK"); + put(HASHTABLEDUMMY, "HASHTABLEDUMMY"); + put(HASHTABLESINK, "HASHTABLESINK"); }}; }