Index: ql/src/test/results/clientpositive/rcfile_createas1.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_createas1.q.out (revision 1199067) +++ ql/src/test/results/clientpositive/rcfile_createas1.q.out (working copy) @@ -60,11 +60,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-6 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-6 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-2 STAGE PLANS: Stage: Stage-1 @@ -88,23 +89,24 @@ table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + name: default.rcfile_createas1b - Stage: Stage-4 + Stage: Stage-5 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true - destination: pfile:/data/users/franklin/hive-block-merge/build/ql/scratchdir/hive_2011-06-09_16-06-50_525_4856448737963146161/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-07_20-01-24_286_8837675491396277390/-ext-10001 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/franklin/hive-block-merge/build/ql/test/data/warehouse/rcfile_createas1b + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/rcfile_createas1b - Stage: Stage-5 + Stage: Stage-6 Create Table Operator: Create Table columns: key int, value string, part int @@ -117,6 +119,9 @@ isExternal: false Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Block level merge @@ -146,7 +151,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_createas1a@ds=1 PREHOOK: Input: default@rcfile_createas1a@ds=2 -PREHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-54_053_5965587433920310393/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_20-01-30_867_1202931125008629355/-mr-10000 POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM rcfile_createas1a @@ -154,7 +159,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_createas1a@ds=1 POSTHOOK: Input: default@rcfile_createas1a@ds=2 -POSTHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-54_053_5965587433920310393/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_20-01-30_867_1202931125008629355/-mr-10000 POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] @@ -166,14 +171,14 @@ ) t PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_createas1b -PREHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-57_460_3734087433150140544/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_20-01-34_389_8689351756025259102/-mr-10000 POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM rcfile_createas1b ) t POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_createas1b -POSTHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-57_460_3734087433150140544/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_20-01-34_389_8689351756025259102/-mr-10000 POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] Index: ql/src/test/results/clientpositive/smb_mapjoin9.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out (revision 1199067) +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out (working copy) @@ -54,7 +54,8 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -109,14 +110,18 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.smb_mapjoin9_results Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/charleschen/hive-trunk/build/ql/test/data/warehouse/smb_mapjoin9_results + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/smb_mapjoin9_results - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: k1 int, value string, ds string, k2 int Index: ql/src/test/results/clientpositive/database.q.out =================================================================== --- ql/src/test/results/clientpositive/database.q.out (revision 1199067) +++ ql/src/test/results/clientpositive/database.q.out (working copy) @@ -138,7 +138,7 @@ POSTHOOK: type: DESCTABLE col1 string -Detailed Table Information Table(tableName:test_table, dbName:test_db, owner:sdong, createTime:1302825475, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/test_db.db/test_table, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825475}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:test_table, dbName:test_db, owner:rsurowka, createTime:1320185179, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/test_db.db/test_table, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185179}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- CREATE LIKE in non-default DB CREATE TABLE test_table_like LIKE test_table PREHOOK: type: CREATETABLE @@ -158,7 +158,7 @@ POSTHOOK: type: DESCTABLE col1 string -Detailed Table Information Table(tableName:test_table_like, dbName:test_db, owner:sdong, createTime:1302825476, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/test_db.db/test_table_like, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825476}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:test_table_like, dbName:test_db, owner:rsurowka, createTime:1320185179, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/test_db.db/test_table_like, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185179}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- LOAD and SELECT LOAD DATA LOCAL INPATH '../data/files/test.dat' OVERWRITE INTO TABLE test_table @@ -172,11 +172,11 @@ PREHOOK: query: SELECT * FROM test_table PREHOOK: type: QUERY PREHOOK: Input: test_db@test_table -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-56_779_6558577544544202317/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-20_901_3351582130660713474/-mr-10000 POSTHOOK: query: SELECT * FROM test_table POSTHOOK: type: QUERY POSTHOOK: Input: test_db@test_table -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-56_779_6558577544544202317/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-20_901_3351582130660713474/-mr-10000 1 2 3 @@ -212,11 +212,11 @@ PREHOOK: query: SELECT * FROM test_table PREHOOK: type: QUERY PREHOOK: Input: test_db@test_table -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_217_230305294028870146/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_559_2863804961457273113/-mr-10000 POSTHOOK: query: SELECT * FROM test_table POSTHOOK: type: QUERY POSTHOOK: Input: test_db@test_table -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_217_230305294028870146/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_559_2863804961457273113/-mr-10000 PREHOOK: query: -- CREATE table that already exists in DEFAULT USE test_db PREHOOK: type: SWITCHDATABASE @@ -238,11 +238,11 @@ PREHOOK: query: SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: test_db@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_462_6058704561576462238/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_877_1007813999079060161/-mr-10000 POSTHOOK: query: SELECT * FROM src LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: test_db@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_462_6058704561576462238/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_877_1007813999079060161/-mr-10000 PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE POSTHOOK: query: USE default @@ -250,11 +250,11 @@ PREHOOK: query: SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_586_3051622495608934510/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-22_018_3313432431075228017/-mr-10000 POSTHOOK: query: SELECT * FROM src LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_586_3051622495608934510/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-22_018_3313432431075228017/-mr-10000 238 val_238 86 val_86 311 val_311 @@ -517,12 +517,12 @@ SELECT * FROM db1.src PREHOOK: type: QUERY PREHOOK: Input: db1@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_137_8852311184592507877/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-29_932_8778703330615338731/-mr-10000 POSTHOOK: query: -- SELECT from foreign table SELECT * FROM db1.src POSTHOOK: type: QUERY POSTHOOK: Input: db1@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_137_8852311184592507877/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-29_932_8778703330615338731/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 238 val_238 86 val_86 @@ -1055,13 +1055,13 @@ WHERE key < 100 AND ds='2008-04-08' AND hr='11' PREHOOK: type: QUERY PREHOOK: Input: db1@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_948_3991317106720358085/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-31_163_2004639451350147676/-mr-10000 POSTHOOK: query: -- SELECT from Partitioned foreign table SELECT key, value FROM db1.srcpart WHERE key < 100 AND ds='2008-04-08' AND hr='11' POSTHOOK: type: QUERY POSTHOOK: Input: db1@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_948_3991317106720358085/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-31_163_2004639451350147676/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 86 val_86 27 val_27 @@ -1159,13 +1159,13 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@src PREHOOK: Input: default@src1 -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-11_120_6942744088220032482/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-34_403_4625698338611139575/-mr-10000 POSTHOOK: query: SELECT a.* FROM db1.src a JOIN default.src1 b ON (a.key = b.key) POSTHOOK: type: QUERY POSTHOOK: Input: db1@src POSTHOOK: Input: default@src1 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-11_120_6942744088220032482/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-34_403_4625698338611139575/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 128 val_128 128 val_128 @@ -1221,6 +1221,7 @@ SELECT value FROM db1.src WHERE key = 8 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: db1@src +Cannot get table db1.db1.conflict_name POSTHOOK: query: -- CREATE foreign table CREATE TABLE db1.conflict_name AS SELECT value FROM db1.src WHERE key = 8 @@ -1237,7 +1238,7 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@conflict_name PREHOOK: Input: db2@conflict_name -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-20_546_4482330222868598236/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-47_662_3710467141209260775/-mr-10000 POSTHOOK: query: -- query tables with the same names in different DBs SELECT * FROM ( SELECT value FROM db1.conflict_name @@ -1247,7 +1248,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: db1@conflict_name POSTHOOK: Input: db2@conflict_name -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-20_546_4482330222868598236/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-47_662_3710467141209260775/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] val_66 val_8 @@ -1264,7 +1265,7 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@conflict_name PREHOOK: Input: db2@conflict_name -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-23_738_1416474990630728009/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-51_126_9069465713985523269/-mr-10000 POSTHOOK: query: SELECT * FROM ( SELECT value FROM db1.conflict_name UNION ALL @@ -1273,7 +1274,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: db1@conflict_name POSTHOOK: Input: db2@conflict_name -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-23_738_1416474990630728009/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-51_126_9069465713985523269/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] val_66 val_8 @@ -1303,11 +1304,11 @@ PREHOOK: query: SELECT key FROM bucketized_src TABLESAMPLE(BUCKET 1 out of 1) PREHOOK: type: QUERY PREHOOK: Input: default@bucketized_src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-31_681_3025368059661975205/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-59_328_3566033008335266406/-mr-10000 POSTHOOK: query: SELECT key FROM bucketized_src TABLESAMPLE(BUCKET 1 out of 1) POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketized_src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-31_681_3025368059661975205/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-59_328_3566033008335266406/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1339,17 +1340,17 @@ key string default value string default -Detailed Table Information Table(tableName:src1, dbName:db2, owner:null, createTime:1302825514, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/db2.db/src1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825514}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:src1, dbName:db2, owner:rsurowka, createTime:1320185222, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/db2.db/src1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185222}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- character escaping SELECT key FROM `default`.src ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-34_830_693129214774849311/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-02_739_7378623196477579765/-mr-10000 POSTHOOK: query: -- character escaping SELECT key FROM `default`.src ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-34_830_693129214774849311/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-02_739_7378623196477579765/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1357,11 +1358,11 @@ PREHOOK: query: SELECT key FROM `default`.`src` ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-37_763_726145307624380559/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-05_934_9221634524540201316/-mr-10000 POSTHOOK: query: SELECT key FROM `default`.`src` ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-37_763_726145307624380559/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-05_934_9221634524540201316/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1369,11 +1370,11 @@ PREHOOK: query: SELECT key FROM default.`src` ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-40_728_8732527538266665217/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-09_167_3309373435458112286/-mr-10000 POSTHOOK: query: SELECT key FROM default.`src` ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-40_728_8732527538266665217/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-09_167_3309373435458112286/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] Index: ql/src/test/results/clientpositive/ctas.q.out =================================================================== --- ql/src/test/results/clientpositive/ctas.q.out (revision 1199067) +++ ql/src/test/results/clientpositive/ctas.q.out (working copy) @@ -6,11 +6,11 @@ PREHOOK: query: select * from nzhang_Tmp PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_tmp -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-51_322_2848708186205887611/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-14_493_1342587779151203373/-mr-10000 POSTHOOK: query: select * from nzhang_Tmp POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_tmp -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-51_322_2848708186205887611/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-14_493_1342587779151203373/-mr-10000 PREHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 @@ -22,7 +22,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -64,7 +65,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-44-51_470_2994705028366987051/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-14_740_6068039279320291425/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -87,14 +88,15 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_CTAS1 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas1 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas1 - Stage: Stage-3 + Stage: Stage-4 Create Table Operator: Create Table columns: k string, value string @@ -105,7 +107,10 @@ name: nzhang_CTAS1 isExternal: false + Stage: Stage-3 + Stats-Aggr Operator + PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -116,11 +121,11 @@ PREHOOK: query: select * from nzhang_CTAS1 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas1 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-58_596_3850188449580676786/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-24_168_1052652908826092056/-mr-10000 POSTHOOK: query: select * from nzhang_CTAS1 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas1 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-58_596_3850188449580676786/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-24_168_1052652908826092056/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -131,6 +136,42 @@ 103 val_103 104 val_104 104 val_104 +PREHOOK: query: describe formatted nzhang_CTAS1 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted nzhang_CTAS1 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +k string None +value string None + +# Detailed Table Information +Database: default +Owner: rsurowka +CreateTime: Mon Nov 07 19:53:23 PST 2011 +LastAccessTime: UNKNOWN +Protect Mode: None +Retention: 0 +Location: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas1 +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 10 + rawDataSize 96 + totalSize 106 + transient_lastDdlTime 1320724404 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas2 as select * from src sort by key, value limit 10 @@ -142,7 +183,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -184,7 +226,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-44-58_860_3982645483317411637/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-24_458_6741209951969271935/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -207,14 +249,15 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_ctas2 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas2 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas2 - Stage: Stage-3 + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string @@ -225,7 +268,10 @@ name: nzhang_ctas2 isExternal: false + Stage: Stage-3 + Stats-Aggr Operator + PREHOOK: query: create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -236,11 +282,11 @@ PREHOOK: query: select * from nzhang_ctas2 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas2 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-05_578_7349012551226510377/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-31_886_3453511835007488493/-mr-10000 POSTHOOK: query: select * from nzhang_ctas2 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas2 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-05_578_7349012551226510377/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-31_886_3453511835007488493/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -251,6 +297,42 @@ 103 val_103 104 val_104 104 val_104 +PREHOOK: query: describe formatted nzhang_CTAS2 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted nzhang_CTAS2 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +Owner: rsurowka +CreateTime: Mon Nov 07 19:53:31 PST 2011 +LastAccessTime: UNKNOWN +Protect Mode: None +Retention: 0 +Location: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas2 +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 10 + rawDataSize 96 + totalSize 106 + transient_lastDdlTime 1320724411 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 @@ -262,7 +344,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -304,7 +387,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-05_893_3271637980302783261/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-32_176_3137057323851157959/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -327,14 +410,15 @@ table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + name: default.nzhang_ctas3 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas3 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas3 - Stage: Stage-3 + Stage: Stage-4 Create Table Operator: Create Table columns: half_key double, conb string @@ -346,7 +430,10 @@ name: nzhang_ctas3 isExternal: false + Stage: Stage-3 + Stats-Aggr Operator + PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -357,11 +444,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_580_9113217289697940221/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-39_570_5842680857812811926/-mr-10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_580_9113217289697940221/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-39_570_5842680857812811926/-mr-10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -372,6 +459,42 @@ 2.5 val_5_con 4.0 val_8_con 4.5 val_9_con +PREHOOK: query: describe formatted nzhang_CTAS3 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted nzhang_CTAS3 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +half_key double None +conb string None + +# Detailed Table Information +Database: default +Owner: rsurowka +CreateTime: Mon Nov 07 19:53:39 PST 2011 +LastAccessTime: UNKNOWN +Protect Mode: None +Retention: 0 +Location: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas3 +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 10 + rawDataSize 120 + totalSize 294 + transient_lastDdlTime 1320724419 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 PREHOOK: type: CREATETABLE POSTHOOK: query: explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2 @@ -390,11 +513,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_947_7015621330413441447/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-39_957_8795808764737341610/-mr-10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_947_7015621330413441447/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-39_957_8795808764737341610/-mr-10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -405,6 +528,42 @@ 2.5 val_5_con 4.0 val_8_con 4.5 val_9_con +PREHOOK: query: describe formatted nzhang_CTAS3 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted nzhang_CTAS3 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +half_key double None +conb string None + +# Detailed Table Information +Database: default +Owner: rsurowka +CreateTime: Mon Nov 07 19:53:39 PST 2011 +LastAccessTime: UNKNOWN +Protect Mode: None +Retention: 0 +Location: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas3 +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 10 + rawDataSize 120 + totalSize 294 + transient_lastDdlTime 1320724419 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 @@ -416,7 +575,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -458,7 +618,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-13_334_919564477125108638/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-40_251_2495661714018508871/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -481,14 +641,15 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_ctas4 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas4 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas4 - Stage: Stage-3 + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string @@ -500,7 +661,10 @@ name: nzhang_ctas4 isExternal: false + Stage: Stage-3 + Stats-Aggr Operator + PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src @@ -511,11 +675,11 @@ PREHOOK: query: select * from nzhang_ctas4 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas4 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-20_050_1736075504443010216/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-47_574_4018493574829459966/-mr-10000 POSTHOOK: query: select * from nzhang_ctas4 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas4 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-20_050_1736075504443010216/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-07_19-53-47_574_4018493574829459966/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -526,6 +690,43 @@ 103 val_103 104 val_104 104 val_104 +PREHOOK: query: describe formatted nzhang_CTAS4 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted nzhang_CTAS4 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +Owner: rsurowka +CreateTime: Mon Nov 07 19:53:47 PST 2011 +LastAccessTime: UNKNOWN +Protect Mode: None +Retention: 0 +Location: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas4 +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numPartitions 0 + numRows 10 + rawDataSize 96 + totalSize 106 + transient_lastDdlTime 1320724427 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim , + serialization.format , PREHOOK: query: explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 @@ -537,7 +738,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-4 STAGE PLANS: Stage: Stage-1 @@ -569,9 +771,9 @@ type: string Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src [src] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src [src] Path -> Partition: - pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -582,12 +784,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src name default.src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1297328964 + transient_lastDdlTime 1320724392 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -598,12 +800,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src name default.src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1297328964 + transient_lastDdlTime 1320724392 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src name: default.src @@ -613,7 +815,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + directory: file:/tmp/rsurowka/hive_2011-11-07_19-53-47_870_2575560272213439831/-mr-10002 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -629,7 +831,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-47_870_2575560272213439831/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -645,9 +847,9 @@ type: string Needs Tagging: false Path -> Alias: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 [file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002] + file:/tmp/rsurowka/hive_2011-11-07_19-53-47_870_2575560272213439831/-mr-10002 [file:/tmp/rsurowka/hive_2011-11-07_19-53-47_870_2575560272213439831/-mr-10002] Path -> Partition: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + file:/tmp/rsurowka/hive_2011-11-07_19-53-47_870_2575560272213439831/-mr-10002 Partition base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -669,9 +871,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-07_19-53-47_870_2575560272213439831/-ext-10001 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-07_19-53-47_870_2575560272213439831/-ext-10001/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -681,19 +883,21 @@ field.delim , line.delim + name default.nzhang_ctas5 serialization.format , + name: default.nzhang_ctas5 TotalFiles: 1 - GatherStats: false + GatherStats: true MultiFileSpray: false Stage: Stage-0 Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001 - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas5 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-07_19-53-47_870_2575560272213439831/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas5 - Stage: Stage-3 + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string @@ -707,7 +911,11 @@ name: nzhang_ctas5 isExternal: false + Stage: Stage-3 + Stats-Aggr Operator + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-07_19-53-47_870_2575560272213439831/-ext-10001/ + PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src Index: ql/src/test/results/clientpositive/merge3.q.out =================================================================== --- ql/src/test/results/clientpositive/merge3.q.out (revision 1199067) +++ ql/src/test/results/clientpositive/merge3.q.out (working copy) @@ -54,11 +54,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-6 + Stage-6 depends on stages: Stage-0 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-2 STAGE PLANS: Stage: Stage-1 @@ -78,24 +79,26 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 TotalFiles: 1 - GatherStats: false + GatherStats: true MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src [merge_src] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src [merge_src] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src Partition base file name: merge_src input format: org.apache.hadoop.mapred.TextInputFormat @@ -106,12 +109,17 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src name default.merge_src + numFiles 4 + numPartitions 0 + numRows 2000 + rawDataSize 21248 serialization.ddl struct merge_src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983697 + totalSize 23248 + transient_lastDdlTime 1320190585 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -122,34 +130,43 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src name default.merge_src + numFiles 4 + numPartitions 0 + numRows 2000 + rawDataSize 21248 serialization.ddl struct merge_src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983697 + totalSize 23248 + transient_lastDdlTime 1320190585 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src name: default.merge_src - Stage: Stage-4 + Stage: Stage-5 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 Stage: Stage-0 Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src2 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src2 - Stage: Stage-5 + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001/ + + Stage: Stage-6 Create Table Operator: Create Table columns: key string, value string @@ -160,14 +177,14 @@ name: merge_src2 isExternal: false - Stage: Stage-2 + Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -175,15 +192,17 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 TotalFiles: 1 GatherStats: false MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -191,6 +210,7 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 input format: org.apache.hadoop.mapred.TextInputFormat @@ -198,7 +218,10 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 + name: default.merge_src2 PREHOOK: query: create table merge_src2 as @@ -217,11 +240,11 @@ PREHOOK: query: select * from merge_src2 PREHOOK: type: QUERY PREHOOK: Input: default@merge_src2 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-01-50_658_5738218994707061399/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-42_710_8938685944801950009/-mr-10000 POSTHOOK: query: select * from merge_src2 POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src2 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-01-50_658_5738218994707061399/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-42_710_8938685944801950009/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] @@ -2280,9 +2303,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-35-46_067_3066864399805575043/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-35-46_067_3066864399805575043/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2292,13 +2315,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1310376946 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -2306,10 +2329,10 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [merge_src_part] - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [merge_src_part] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2322,7 +2345,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 name default.merge_src_part numFiles 2 numPartitions 2 @@ -2333,7 +2356,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2344,7 +2367,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -2355,11 +2378,11 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2372,7 +2395,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 name default.merge_src_part numFiles 2 numPartitions 2 @@ -2383,7 +2406,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2394,7 +2417,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -2405,7 +2428,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part @@ -2417,8 +2440,8 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 Stage: Stage-0 Move Operator @@ -2426,7 +2449,7 @@ partition: ds replace: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2436,29 +2459,29 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 - tmp directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10001 + tmp directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -2469,13 +2492,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -2483,9 +2506,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2496,13 +2519,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2513,13 +2536,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 name: default.merge_src_part2 @@ -2566,12 +2589,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@merge_src_part2@ds=2008-04-08 PREHOOK: Input: default@merge_src_part2@ds=2008-04-09 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-00_759_2190604757532214555/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-53_907_7127634955696857565/-mr-10000 POSTHOOK: query: select * from merge_src_part2 where ds is not null POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src_part2@ds=2008-04-08 POSTHOOK: Input: default@merge_src_part2@ds=2008-04-09 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-00_759_2190604757532214555/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-53_907_7127634955696857565/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] @@ -4670,10 +4693,10 @@ type: string Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [s:merge_src_part] - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [s:merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [s:merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [s:merge_src_part] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4686,7 +4709,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 name default.merge_src_part numFiles 2 numPartitions 2 @@ -4697,7 +4720,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4708,7 +4731,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -4719,11 +4742,11 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4736,7 +4759,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 name default.merge_src_part numFiles 2 numPartitions 2 @@ -4747,7 +4770,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4758,7 +4781,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -4769,7 +4792,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part @@ -4787,9 +4810,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -4799,13 +4822,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -4819,8 +4842,8 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 Stage: Stage-0 Move Operator @@ -4828,7 +4851,7 @@ partition: ds replace: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -4838,29 +4861,29 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 - tmp directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10001 + tmp directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -4871,13 +4894,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -4885,9 +4908,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4898,13 +4921,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4915,13 +4938,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 name: default.merge_src_part2 @@ -4976,12 +4999,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@merge_src_part2@ds=2008-04-08 PREHOOK: Input: default@merge_src_part2@ds=2008-04-09 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-09_192_4139404353176082441/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-37-02_696_6879214383302578391/-mr-10000 POSTHOOK: query: select * from merge_src_part2 where ds is not null POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src_part2@ds=2008-04-08 POSTHOOK: Input: default@merge_src_part2@ds=2008-04-09 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-09_192_4139404353176082441/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-37-02_696_6879214383302578391/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] Index: ql/src/test/queries/clientpositive/ctas.q =================================================================== --- ql/src/test/queries/clientpositive/ctas.q (revision 1199067) +++ ql/src/test/queries/clientpositive/ctas.q (working copy) @@ -15,34 +15,44 @@ select * from nzhang_CTAS1; +describe formatted nzhang_CTAS1; + explain create table nzhang_ctas2 as select * from src sort by key, value limit 10; create table nzhang_ctas2 as select * from src sort by key, value limit 10; select * from nzhang_ctas2; +describe formatted nzhang_CTAS2; + explain create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10; create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10; select * from nzhang_ctas3; +describe formatted nzhang_CTAS3; + explain create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2; create table if not exists nzhang_ctas3 as select key, value from src sort by key, value limit 2; select * from nzhang_ctas3; +describe formatted nzhang_CTAS3; + explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10; create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10; select * from nzhang_ctas4; +describe formatted nzhang_CTAS4; + explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10; set mapred.job.tracker=does.notexist.com:666; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -167,7 +167,14 @@ Task currTask, HiveConf hconf) { MoveWork mvWork = ((MoveTask)mvTask).getWork(); - StatsWork statsWork = new StatsWork(mvWork.getLoadTableWork()); + StatsWork statsWork = null; + if(mvWork.getLoadTableWork() != null){ + statsWork = new StatsWork(mvWork.getLoadTableWork()); + }else if (mvWork.getLoadFileWork() != null){ + statsWork = new StatsWork(mvWork.getLoadFileWork()); + } + assert statsWork != null : "Error when genereting StatsTask"; + MapredWork mrWork = (MapredWork) currTask.getWork(); // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix Index: ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (working copy) @@ -216,16 +216,27 @@ public int execute(DriverContext driverContext) { LOG.info("Executing stats task"); - // Make sure that it is either an ANALYZE command or an INSERT OVERWRITE command - assert (work.getLoadTableDesc() != null && work.getTableSpecs() == null || work - .getLoadTableDesc() == null && work.getTableSpecs() != null); + // Make sure that it is either an ANALYZE, INSERT OVERWRITE or CTAS command + short workComponentsPresent = 0; + if (work.getLoadTableDesc() != null) + workComponentsPresent++; + if (work.getTableSpecs() != null) + workComponentsPresent++; + if (work.getLoadFileDesc() != null) + workComponentsPresent++; + + assert (workComponentsPresent == 1); + String tableName = ""; try { if (work.getLoadTableDesc() != null) { tableName = work.getLoadTableDesc().getTable().getTableName(); + } else if (work.getTableSpecs() != null){ + tableName = work.getTableSpecs().tableName; } else { - tableName = work.getTableSpecs().tableName; + tableName = work.getLoadFileDesc().getDestinationCreateTable(); } + table = db.getTable(tableName); } catch (HiveException e) { @@ -292,7 +303,7 @@ List partitions = getPartitionsList(); boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); - if (partitions == null) { + if (partitions == null || partitions.size() == 0) { // non-partitioned tables: if (!tableStatsExist && atomic) { return 0; @@ -310,7 +321,7 @@ // In case of a non-partitioned table, the key for stats temporary store is "rootDir" if (statsAggregator != null) { - updateStats(collectableStats, tblStats, statsAggregator, parameters, + updateStats(collectableStats, tblStats, statsAggregator, parameters, work.getAggKey(), atomic); statsAggregator.cleanUp(work.getAggKey()); } @@ -349,7 +360,7 @@ LOG.info("Stats aggregator : " + partitionID); if (statsAggregator != null) { - updateStats(collectableStats, newPartStats, statsAggregator, + updateStats(collectableStats, newPartStats, statsAggregator, parameters, partitionID, atomic); } else { for (String statType : collectableStats) { @@ -447,7 +458,7 @@ if (value != null) { longValue = Long.parseLong(value); - if (work.getLoadTableDesc() != null && + if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null) { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (working copy) @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,6 +34,7 @@ @Explain(displayName = "Create Table") public class CreateTableDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 1L; + String databaseName; String tableName; boolean isExternal; ArrayList cols; @@ -60,6 +60,26 @@ public CreateTableDesc() { } + public CreateTableDesc(String databaseName, String tableName, boolean isExternal, + List cols, List partCols, + List bucketCols, List sortCols, int numBuckets, + String fieldDelim, String fieldEscape, String collItemDelim, + String mapKeyDelim, String lineDelim, String comment, String inputFormat, + String outputFormat, String location, String serName, + String storageHandler, + Map serdeProps, + Map tblProps, + boolean ifNotExists) { + + this(tableName, isExternal, cols, partCols, + bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape, + collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat, + outputFormat, location, serName, storageHandler, serdeProps, + tblProps, ifNotExists); + + this.databaseName = databaseName; + } + public CreateTableDesc(String tableName, boolean isExternal, List cols, List partCols, List bucketCols, List sortCols, int numBuckets, @@ -117,6 +137,10 @@ return tableName; } + public String getDatabaseName(){ + return databaseName; + } + public void setTableName(String tableName) { this.tableName = tableName; } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (working copy) @@ -32,8 +32,9 @@ private tableSpec tableSpecs; // source table spec -- for TableScanOperator private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator + private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator private String aggKey; // aggregation key prefix - + private boolean noStatsAggregator = false; public StatsWork() { @@ -47,6 +48,10 @@ this.loadTableDesc = loadTableDesc; } + public StatsWork(LoadFileDesc loadFileDesc) { + this.loadFileDesc = loadFileDesc; + } + public tableSpec getTableSpecs() { return tableSpecs; } @@ -55,6 +60,10 @@ return loadTableDesc; } + public LoadFileDesc getLoadFileDesc() { + return loadFileDesc; + } + public void setAggKey(String aggK) { aggKey = aggK; } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -263,6 +263,11 @@ properties.setProperty(Constants.LINE_DELIM, crtTblDesc.getLineDelim()); } + if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + crtTblDesc.getDatabaseName() + "." + crtTblDesc.getTableName()); + } + // replace the default input & output file format with those found in // crtTblDesc Class c1 = Class.forName(crtTblDesc.getInputFormat()); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java (working copy) @@ -31,10 +31,19 @@ // list of columns, comma separated private String columns; private String columnTypes; + private String destinationCreateTable; public LoadFileDesc() { } + public LoadFileDesc(final CreateTableDesc createTableDesc, final String sourceDir, final String targetDir, + final boolean isDfsDir, final String columns, final String columnTypes) { + this(sourceDir, targetDir, isDfsDir, columns, columnTypes); + if (createTableDesc != null && createTableDesc.getDatabaseName() != null && createTableDesc.getTableName() != null){ + destinationCreateTable = createTableDesc.getDatabaseName() + "." + createTableDesc.getTableName(); + } + } + public LoadFileDesc(final String sourceDir, final String targetDir, final boolean isDfsDir, final String columns, final String columnTypes) { @@ -92,4 +101,11 @@ public void setColumnTypes(String columnTypes) { this.columnTypes = columnTypes; } + + /** + * @return the destinationCreateTable + */ + public String getDestinationCreateTable(){ + return destinationCreateTable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -619,7 +619,8 @@ assert (ast.getToken().getType() == HiveParser.TOK_TAB || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION - || ast.getToken().getType() == HiveParser.TOK_TABTYPE); + || ast.getToken().getType() == HiveParser.TOK_TABTYPE + || ast.getToken().getType() == HiveParser.TOK_CREATETABLE); int childIndex = 0; numDynParts = 0; @@ -631,8 +632,9 @@ tableName = conf.getVar(HiveConf.ConfVars.HIVETESTMODEPREFIX) + tableName; } - - tableHandle = db.getTable(tableName); + if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + tableHandle = db.getTable(tableName); + } } catch (InvalidTableException ite) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(ast .getChild(0)), ite); @@ -642,7 +644,7 @@ } // get partition metadata if partition specified - if (ast.getChildCount() == 2) { + if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { childIndex = 1; ASTNode partspec = (ASTNode) ast.getChild(1); partitions = new ArrayList(); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1199067) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -27,9 +27,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -94,6 +95,7 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3; @@ -103,14 +105,10 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory; import org.apache.hadoop.hive.ql.optimizer.Optimizer; -import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeCreateCommonVars; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.RowFormatParams; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.StorageFormat; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; @@ -127,6 +125,7 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.HiveOperation; @@ -149,13 +148,12 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -1053,6 +1051,13 @@ throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location), e); } + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + tableSpec ts = new tableSpec(db, conf, this.ast); + // Set that variable to automatically collect stats during the MapReduce job + qb.getParseInfo().setIsInsertToTable(true); + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); + } } else { qb.setIsQuery(true); fname = ctx.getMRTmpFileURI(); @@ -3994,7 +3999,7 @@ } boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE); - loadFileWork.add(new LoadFileDesc(queryTmpdir, destStr, isDfsDir, cols, + loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, destStr, isDfsDir, cols, colTypes)); if (tblDesc == null) { @@ -6920,6 +6925,10 @@ Path targetPath; try { dumpTable = db.newTable(qb.getTableDesc().getTableName()); + if (!db.databaseExists(dumpTable.getDbName())) { + throw new SemanticException("ERROR: The database " + dumpTable.getDbName() + + " does not exist."); + } Warehouse wh = new Warehouse(conf); targetPath = wh.getTablePath(db.getDatabase(dumpTable.getDbName()), dumpTable .getTableName()); @@ -7034,7 +7043,18 @@ getLeafTasks(rootTasks, leaves); assert (leaves.size() > 0); for (Task task : leaves) { - task.addDependentTask(crtTblTask); + if (task instanceof StatsTask){ + //StatsTask require table to already exist + for (Task parentOfStatsTask : task.getParentTasks()){ + parentOfStatsTask.addDependentTask(crtTblTask); + } + for (Task parentOfCrtTblTask : crtTblTask.getParentTasks()){ + parentOfCrtTblTask.removeDependentTask(task); + } + crtTblTask.addDependentTask(task); + } else { + task.addDependentTask(crtTblTask); + } } } @@ -7867,8 +7887,10 @@ case CTAS: // create table as select // Verify that the table does not already exist + String databaseName; try { Table dumpTable = db.newTable(tableName); + databaseName = dumpTable.getDbName(); if (null == db.getDatabase(dumpTable.getDbName()) ) { throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(dumpTable.getDbName())); } @@ -7881,7 +7903,7 @@ tblProps = addDefaultProperties(tblProps); - crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols, + crtTblDesc = new CreateTableDesc(databaseName, tableName, isExt, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, storageFormat.storageHandler, shared.serdeProps,