Index: ql/src/test/results/clientpositive/rcfile_createas1.q.out =================================================================== --- ql/src/test/results/clientpositive/rcfile_createas1.q.out (revision 1196269) +++ ql/src/test/results/clientpositive/rcfile_createas1.q.out (working copy) @@ -60,11 +60,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-6 + Stage-6 depends on stages: Stage-0 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-2 STAGE PLANS: Stage: Stage-1 @@ -88,23 +89,27 @@ table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + name: default.rcfile_createas1b - Stage: Stage-4 + Stage: Stage-5 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true - destination: pfile:/data/users/franklin/hive-block-merge/build/ql/scratchdir/hive_2011-06-09_16-06-50_525_4856448737963146161/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-10-26_17-46-22_973_2711014416100036624/-ext-10001 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/franklin/hive-block-merge/build/ql/test/data/warehouse/rcfile_createas1b + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/rcfile_createas1b - Stage: Stage-5 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-6 Create Table Operator: Create Table columns: key int, value string, part int @@ -116,7 +121,7 @@ name: rcfile_createas1b isExternal: false - Stage: Stage-2 + Stage: Stage-3 Block level merge @@ -146,7 +151,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_createas1a@ds=1 PREHOOK: Input: default@rcfile_createas1a@ds=2 -PREHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-54_053_5965587433920310393/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-46-28_244_4573733618566571997/-mr-10000 POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM rcfile_createas1a @@ -154,7 +159,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_createas1a@ds=1 POSTHOOK: Input: default@rcfile_createas1a@ds=2 -POSTHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-54_053_5965587433920310393/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-46-28_244_4573733618566571997/-mr-10000 POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] @@ -166,14 +171,14 @@ ) t PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_createas1b -PREHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-57_460_3734087433150140544/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-46-31_675_7204189127600594588/-mr-10000 POSTHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM rcfile_createas1b ) t POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_createas1b -POSTHOOK: Output: file:/tmp/franklin/hive_2011-06-09_16-06-57_460_3734087433150140544/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-46-31_675_7204189127600594588/-mr-10000 POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: rcfile_createas1a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] Index: ql/src/test/results/clientpositive/smb_mapjoin9.q.out =================================================================== --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out (revision 1196269) +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out (working copy) @@ -54,7 +54,8 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -109,14 +110,18 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.smb_mapjoin9_results Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/charleschen/hive-trunk/build/ql/test/data/warehouse/smb_mapjoin9_results + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/smb_mapjoin9_results - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: k1 int, value string, ds string, k2 int Index: ql/src/test/results/clientpositive/database.q.out =================================================================== --- ql/src/test/results/clientpositive/database.q.out (revision 1196269) +++ ql/src/test/results/clientpositive/database.q.out (working copy) @@ -138,7 +138,7 @@ POSTHOOK: type: DESCTABLE col1 string -Detailed Table Information Table(tableName:test_table, dbName:test_db, owner:sdong, createTime:1302825475, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/test_db.db/test_table, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825475}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:test_table, dbName:test_db, owner:rsurowka, createTime:1320185179, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/test_db.db/test_table, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185179}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- CREATE LIKE in non-default DB CREATE TABLE test_table_like LIKE test_table PREHOOK: type: CREATETABLE @@ -158,7 +158,7 @@ POSTHOOK: type: DESCTABLE col1 string -Detailed Table Information Table(tableName:test_table_like, dbName:test_db, owner:sdong, createTime:1302825476, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/test_db.db/test_table_like, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825476}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:test_table_like, dbName:test_db, owner:rsurowka, createTime:1320185179, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:col1, type:string, comment:null)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/test_db.db/test_table_like, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185179}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- LOAD and SELECT LOAD DATA LOCAL INPATH '../data/files/test.dat' OVERWRITE INTO TABLE test_table @@ -172,11 +172,11 @@ PREHOOK: query: SELECT * FROM test_table PREHOOK: type: QUERY PREHOOK: Input: test_db@test_table -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-56_779_6558577544544202317/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-20_901_3351582130660713474/-mr-10000 POSTHOOK: query: SELECT * FROM test_table POSTHOOK: type: QUERY POSTHOOK: Input: test_db@test_table -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-56_779_6558577544544202317/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-20_901_3351582130660713474/-mr-10000 1 2 3 @@ -212,11 +212,11 @@ PREHOOK: query: SELECT * FROM test_table PREHOOK: type: QUERY PREHOOK: Input: test_db@test_table -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_217_230305294028870146/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_559_2863804961457273113/-mr-10000 POSTHOOK: query: SELECT * FROM test_table POSTHOOK: type: QUERY POSTHOOK: Input: test_db@test_table -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_217_230305294028870146/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_559_2863804961457273113/-mr-10000 PREHOOK: query: -- CREATE table that already exists in DEFAULT USE test_db PREHOOK: type: SWITCHDATABASE @@ -238,11 +238,11 @@ PREHOOK: query: SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: test_db@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_462_6058704561576462238/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_877_1007813999079060161/-mr-10000 POSTHOOK: query: SELECT * FROM src LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: test_db@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_462_6058704561576462238/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-21_877_1007813999079060161/-mr-10000 PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE POSTHOOK: query: USE default @@ -250,11 +250,11 @@ PREHOOK: query: SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_586_3051622495608934510/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-22_018_3313432431075228017/-mr-10000 POSTHOOK: query: SELECT * FROM src LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-57-58_586_3051622495608934510/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-22_018_3313432431075228017/-mr-10000 238 val_238 86 val_86 311 val_311 @@ -517,12 +517,12 @@ SELECT * FROM db1.src PREHOOK: type: QUERY PREHOOK: Input: db1@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_137_8852311184592507877/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-29_932_8778703330615338731/-mr-10000 POSTHOOK: query: -- SELECT from foreign table SELECT * FROM db1.src POSTHOOK: type: QUERY POSTHOOK: Input: db1@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_137_8852311184592507877/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-29_932_8778703330615338731/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 238 val_238 86 val_86 @@ -1055,13 +1055,13 @@ WHERE key < 100 AND ds='2008-04-08' AND hr='11' PREHOOK: type: QUERY PREHOOK: Input: db1@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_948_3991317106720358085/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-31_163_2004639451350147676/-mr-10000 POSTHOOK: query: -- SELECT from Partitioned foreign table SELECT key, value FROM db1.srcpart WHERE key < 100 AND ds='2008-04-08' AND hr='11' POSTHOOK: type: QUERY POSTHOOK: Input: db1@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-07_948_3991317106720358085/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-31_163_2004639451350147676/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 86 val_86 27 val_27 @@ -1159,13 +1159,13 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@src PREHOOK: Input: default@src1 -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-11_120_6942744088220032482/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-34_403_4625698338611139575/-mr-10000 POSTHOOK: query: SELECT a.* FROM db1.src a JOIN default.src1 b ON (a.key = b.key) POSTHOOK: type: QUERY POSTHOOK: Input: db1@src POSTHOOK: Input: default@src1 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-11_120_6942744088220032482/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-34_403_4625698338611139575/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] 128 val_128 128 val_128 @@ -1221,6 +1221,7 @@ SELECT value FROM db1.src WHERE key = 8 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: db1@src +Cannot get table db1.db1.conflict_name POSTHOOK: query: -- CREATE foreign table CREATE TABLE db1.conflict_name AS SELECT value FROM db1.src WHERE key = 8 @@ -1237,7 +1238,7 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@conflict_name PREHOOK: Input: db2@conflict_name -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-20_546_4482330222868598236/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-47_662_3710467141209260775/-mr-10000 POSTHOOK: query: -- query tables with the same names in different DBs SELECT * FROM ( SELECT value FROM db1.conflict_name @@ -1247,7 +1248,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: db1@conflict_name POSTHOOK: Input: db2@conflict_name -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-20_546_4482330222868598236/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-47_662_3710467141209260775/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] val_66 val_8 @@ -1264,7 +1265,7 @@ PREHOOK: type: QUERY PREHOOK: Input: db1@conflict_name PREHOOK: Input: db2@conflict_name -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-23_738_1416474990630728009/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-51_126_9069465713985523269/-mr-10000 POSTHOOK: query: SELECT * FROM ( SELECT value FROM db1.conflict_name UNION ALL @@ -1273,7 +1274,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: db1@conflict_name POSTHOOK: Input: db2@conflict_name -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-23_738_1416474990630728009/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-51_126_9069465713985523269/-mr-10000 POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] val_66 val_8 @@ -1303,11 +1304,11 @@ PREHOOK: query: SELECT key FROM bucketized_src TABLESAMPLE(BUCKET 1 out of 1) PREHOOK: type: QUERY PREHOOK: Input: default@bucketized_src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-31_681_3025368059661975205/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-59_328_3566033008335266406/-mr-10000 POSTHOOK: query: SELECT key FROM bucketized_src TABLESAMPLE(BUCKET 1 out of 1) POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketized_src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-31_681_3025368059661975205/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-06-59_328_3566033008335266406/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1339,17 +1340,17 @@ key string default value string default -Detailed Table Information Table(tableName:src1, dbName:db2, owner:null, createTime:1302825514, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/sdong/www/open-source-hive3/build/ql/test/data/warehouse/db2.db/src1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1302825514}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) +Detailed Table Information Table(tableName:src1, dbName:db2, owner:rsurowka, createTime:1320185222, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/db2.db/src1, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1320185222}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE) PREHOOK: query: -- character escaping SELECT key FROM `default`.src ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-34_830_693129214774849311/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-02_739_7378623196477579765/-mr-10000 POSTHOOK: query: -- character escaping SELECT key FROM `default`.src ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-34_830_693129214774849311/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-02_739_7378623196477579765/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1357,11 +1358,11 @@ PREHOOK: query: SELECT key FROM `default`.`src` ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-37_763_726145307624380559/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-05_934_9221634524540201316/-mr-10000 POSTHOOK: query: SELECT key FROM `default`.`src` ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-37_763_726145307624380559/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-05_934_9221634524540201316/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] @@ -1369,11 +1370,11 @@ PREHOOK: query: SELECT key FROM default.`src` ORDER BY key LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-40_728_8732527538266665217/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-09_167_3309373435458112286/-mr-10000 POSTHOOK: query: SELECT key FROM default.`src` ORDER BY key LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/tmp/sdong/hive_2011-04-14_16-58-40_728_8732527538266665217/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_15-07-09_167_3309373435458112286/-mr-10000 POSTHOOK: Lineage: bucketized_src.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: bucketized_src.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: temp_tbl2.c EXPRESSION [(temp_tbl)temp_tbl.null, ] Index: ql/src/test/results/clientpositive/ctas.q.out =================================================================== --- ql/src/test/results/clientpositive/ctas.q.out (revision 1196269) +++ ql/src/test/results/clientpositive/ctas.q.out (working copy) @@ -6,11 +6,11 @@ PREHOOK: query: select * from nzhang_Tmp PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_tmp -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-51_322_2848708186205887611/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-14_089_7805371435055168088/-mr-10000 POSTHOOK: query: select * from nzhang_Tmp POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_tmp -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-51_322_2848708186205887611/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-14_089_7805371435055168088/-mr-10000 PREHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: query: explain create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 @@ -22,7 +22,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -64,7 +65,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-44-51_470_2994705028366987051/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-14_320_2857131931677646048/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -87,14 +88,18 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_CTAS1 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas1 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas1 Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: k string, value string @@ -116,11 +121,11 @@ PREHOOK: query: select * from nzhang_CTAS1 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas1 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-58_596_3850188449580676786/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-22_149_2550900719072263257/-mr-10000 POSTHOOK: query: select * from nzhang_CTAS1 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas1 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-44-58_596_3850188449580676786/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-22_149_2550900719072263257/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -142,7 +147,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -184,7 +190,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-44-58_860_3982645483317411637/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-22_365_1046221914527264569/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -207,14 +213,18 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_ctas2 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas2 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas2 Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string @@ -236,11 +246,11 @@ PREHOOK: query: select * from nzhang_ctas2 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas2 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-05_578_7349012551226510377/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-28_677_4539565454471520072/-mr-10000 POSTHOOK: query: select * from nzhang_ctas2 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas2 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-05_578_7349012551226510377/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-28_677_4539565454471520072/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -262,7 +272,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -304,7 +315,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-05_893_3271637980302783261/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-28_863_7160930259157065839/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -327,14 +338,18 @@ table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + name: default.nzhang_ctas3 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas3 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas3 Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: half_key double, conb string @@ -357,11 +372,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_580_9113217289697940221/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-35_199_1508600280549722665/-mr-10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_580_9113217289697940221/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-35_199_1508600280549722665/-mr-10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -390,11 +405,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_947_7015621330413441447/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-35_478_2177108315743457713/-mr-10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-12_947_7015621330413441447/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-35_478_2177108315743457713/-mr-10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -416,7 +431,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -458,7 +474,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-13_334_919564477125108638/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-35_671_5760430873302088908/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -481,14 +497,18 @@ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + name: default.nzhang_ctas4 Stage: Stage-0 Move Operator files: hdfs directory: true - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas4 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas4 Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string @@ -511,11 +531,11 @@ PREHOOK: query: select * from nzhang_ctas4 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas4 -PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-20_050_1736075504443010216/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-41_951_6389725926790961556/-mr-10000 POSTHOOK: query: select * from nzhang_ctas4 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas4 -POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-45-20_050_1736075504443010216/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-10-26_17-41-41_951_6389725926790961556/-mr-10000 0 val_0 0 val_0 0 val_0 @@ -537,7 +557,8 @@ Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -569,9 +590,9 @@ type: string Needs Tagging: false Path -> Alias: - pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src [src] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src [src] Path -> Partition: - pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -582,12 +603,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src name default.src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1297328964 + transient_lastDdlTime 1319676071 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -598,12 +619,12 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/src name default.src serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1297328964 + transient_lastDdlTime 1319676071 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src name: default.src @@ -613,7 +634,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + directory: file:/tmp/rsurowka/hive_2011-10-26_17-41-42_135_3160914753422712523/-mr-10002 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -629,7 +650,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-42_135_3160914753422712523/-mr-10002 Reduce Output Operator key expressions: expr: _col0 @@ -645,9 +666,9 @@ type: string Needs Tagging: false Path -> Alias: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 [file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002] + file:/tmp/rsurowka/hive_2011-10-26_17-41-42_135_3160914753422712523/-mr-10002 [file:/tmp/rsurowka/hive_2011-10-26_17-41-42_135_3160914753422712523/-mr-10002] Path -> Partition: - file:/tmp/sdong/hive_2011-02-10_01-45-20_305_5281789596105491506/-mr-10002 + file:/tmp/rsurowka/hive_2011-10-26_17-41-42_135_3160914753422712523/-mr-10002 Partition base file name: -mr-10002 input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -669,9 +690,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-10-26_17-41-42_135_3160914753422712523/-ext-10001 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-10-26_17-41-42_135_3160914753422712523/-ext-10001/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -681,19 +702,25 @@ field.delim , line.delim + name default.nzhang_ctas5 serialization.format , + name: default.nzhang_ctas5 TotalFiles: 1 - GatherStats: false + GatherStats: true MultiFileSpray: false Stage: Stage-0 Move Operator files: hdfs directory: true - source: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_01-45-20_305_5281789596105491506/-ext-10001 - destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/nzhang_ctas5 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-10-26_17-41-42_135_3160914753422712523/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/nzhang_ctas5 Stage: Stage-3 + Stats-Aggr Operator + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-10-26_17-41-42_135_3160914753422712523/-ext-10001/ + + Stage: Stage-4 Create Table Operator: Create Table columns: key string, value string Index: ql/src/test/results/clientpositive/merge3.q.out =================================================================== --- ql/src/test/results/clientpositive/merge3.q.out (revision 1196269) +++ ql/src/test/results/clientpositive/merge3.q.out (working copy) @@ -54,11 +54,12 @@ STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-4 depends on stages: Stage-1 , consists of Stage-3, Stage-2 + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-6 + Stage-6 depends on stages: Stage-0 Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-2 STAGE PLANS: Stage: Stage-1 @@ -78,24 +79,26 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 TotalFiles: 1 - GatherStats: false + GatherStats: true MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src [merge_src] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src [merge_src] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src Partition base file name: merge_src input format: org.apache.hadoop.mapred.TextInputFormat @@ -106,12 +109,17 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src name default.merge_src + numFiles 4 + numPartitions 0 + numRows 2000 + rawDataSize 21248 serialization.ddl struct merge_src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983697 + totalSize 23248 + transient_lastDdlTime 1320190585 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -122,34 +130,43 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src name default.merge_src + numFiles 4 + numPartitions 0 + numRows 2000 + rawDataSize 21248 serialization.ddl struct merge_src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983697 + totalSize 23248 + transient_lastDdlTime 1320190585 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src name: default.merge_src - Stage: Stage-4 + Stage: Stage-5 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 Stage: Stage-0 Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src2 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src2 - Stage: Stage-5 + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001/ + + Stage: Stage-6 Create Table Operator: Create Table columns: key string, value string @@ -160,14 +177,14 @@ name: merge_src2 isExternal: false - Stage: Stage-2 + Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10001 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10001 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -175,15 +192,17 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 TotalFiles: 1 GatherStats: false MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-44_376_1447687532118180204/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-33_667_5823725532103005347/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -191,6 +210,7 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 input format: org.apache.hadoop.mapred.TextInputFormat @@ -198,7 +218,10 @@ properties: columns _col0,_col1 columns.types string:string + name default.merge_src2 serialization.format 1 + name: default.merge_src2 + name: default.merge_src2 PREHOOK: query: create table merge_src2 as @@ -217,11 +240,11 @@ PREHOOK: query: select * from merge_src2 PREHOOK: type: QUERY PREHOOK: Input: default@merge_src2 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-01-50_658_5738218994707061399/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-42_710_8938685944801950009/-mr-10000 POSTHOOK: query: select * from merge_src2 POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src2 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-01-50_658_5738218994707061399/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-42_710_8938685944801950009/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] @@ -2280,9 +2303,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-35-46_067_3066864399805575043/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-35-46_067_3066864399805575043/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2292,13 +2315,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1310376946 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -2306,10 +2329,10 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [merge_src_part] - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [merge_src_part] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2322,7 +2345,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 name default.merge_src_part numFiles 2 numPartitions 2 @@ -2333,7 +2356,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2344,7 +2367,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -2355,11 +2378,11 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2372,7 +2395,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 name default.merge_src_part numFiles 2 numPartitions 2 @@ -2383,7 +2406,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2394,7 +2417,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -2405,7 +2428,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part @@ -2417,8 +2440,8 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 Stage: Stage-0 Move Operator @@ -2426,7 +2449,7 @@ partition: ds replace: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2436,29 +2459,29 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 - tmp directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10001 + tmp directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10000 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -2469,13 +2492,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -2483,9 +2506,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-01-51_051_6120737160044680268/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-43_140_7904421244841722338/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -2496,13 +2519,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -2513,13 +2536,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983711 + transient_lastDdlTime 1320190603 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 name: default.merge_src_part2 @@ -2566,12 +2589,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@merge_src_part2@ds=2008-04-08 PREHOOK: Input: default@merge_src_part2@ds=2008-04-09 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-00_759_2190604757532214555/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-53_907_7127634955696857565/-mr-10000 POSTHOOK: query: select * from merge_src_part2 where ds is not null POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src_part2@ds=2008-04-08 POSTHOOK: Input: default@merge_src_part2@ds=2008-04-09 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-00_759_2190604757532214555/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-36-53_907_7127634955696857565/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] @@ -4670,10 +4693,10 @@ type: string Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [s:merge_src_part] - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [s:merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 [s:merge_src_part] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 [s:merge_src_part] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 Partition base file name: ds=2008-04-08 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4686,7 +4709,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-08 name default.merge_src_part numFiles 2 numPartitions 2 @@ -4697,7 +4720,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4708,7 +4731,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -4719,11 +4742,11 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part - pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 Partition base file name: ds=2008-04-09 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4736,7 +4759,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part/ds=2008-04-09 name default.merge_src_part numFiles 2 numPartitions 2 @@ -4747,7 +4770,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 11624 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4758,7 +4781,7 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part name default.merge_src_part numFiles 4 numPartitions 2 @@ -4769,7 +4792,7 @@ serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe totalSize 23248 - transient_lastDdlTime 1306983704 + transient_lastDdlTime 1320190593 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part name: default.merge_src_part @@ -4787,9 +4810,9 @@ File Output Operator compressed: false GlobalTableId: 1 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 NumFilesPerFileSink: 1 - Stats Publishing Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000/ + Stats Publishing Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -4799,13 +4822,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -4819,8 +4842,8 @@ Move Operator files: hdfs directory: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 - destination: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 + destination: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 Stage: Stage-0 Move Operator @@ -4828,7 +4851,7 @@ partition: ds replace: true - source: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + source: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -4838,29 +4861,29 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 - tmp directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10001 + tmp directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10001 Stage: Stage-2 Stats-Aggr Operator - Stats Aggregation Key Prefix: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000/ + Stats Aggregation Key Prefix: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 - directory: pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10000 + directory: pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -4871,13 +4894,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 TotalFiles: 1 @@ -4885,9 +4908,9 @@ MultiFileSpray: false Needs Tagging: false Path -> Alias: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 [pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002] + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 [pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002] Path -> Partition: - pfile:/data/users/tomasz/apache-hive/build/ql/scratchdir/hive_2011-06-01_20-02-01_691_3177939093965437064/-ext-10002 + pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/scratchdir/hive_2011-11-01_16-36-54_793_7995735349497156129/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat @@ -4898,13 +4921,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat @@ -4915,13 +4938,13 @@ columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location pfile:/data/users/tomasz/apache-hive/build/ql/test/data/warehouse/merge_src_part2 + location pfile:/data/users/rsurowka/JAVA_HIVE/apache-hive/build/ql/test/data/warehouse/merge_src_part2 name default.merge_src_part2 partition_columns ds serialization.ddl struct merge_src_part2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - transient_lastDdlTime 1306983721 + transient_lastDdlTime 1320190614 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.merge_src_part2 name: default.merge_src_part2 @@ -4976,12 +4999,12 @@ PREHOOK: type: QUERY PREHOOK: Input: default@merge_src_part2@ds=2008-04-08 PREHOOK: Input: default@merge_src_part2@ds=2008-04-09 -PREHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-09_192_4139404353176082441/-mr-10000 +PREHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-37-02_696_6879214383302578391/-mr-10000 POSTHOOK: query: select * from merge_src_part2 where ds is not null POSTHOOK: type: QUERY POSTHOOK: Input: default@merge_src_part2@ds=2008-04-08 POSTHOOK: Input: default@merge_src_part2@ds=2008-04-09 -POSTHOOK: Output: file:/tmp/tomasz/hive_2011-06-01_20-02-09_192_4139404353176082441/-mr-10000 +POSTHOOK: Output: file:/tmp/rsurowka/hive_2011-11-01_16-37-02_696_6879214383302578391/-mr-10000 POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: merge_src_part PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -167,7 +167,14 @@ Task currTask, HiveConf hconf) { MoveWork mvWork = ((MoveTask)mvTask).getWork(); - StatsWork statsWork = new StatsWork(mvWork.getLoadTableWork()); + StatsWork statsWork = null; + if(mvWork.getLoadTableWork() != null){ + statsWork = new StatsWork(mvWork.getLoadTableWork()); + }else if (mvWork.getLoadFileWork() != null){ + statsWork = new StatsWork(mvWork.getLoadFileWork()); + } + assert statsWork != null : "Error when genereting StatsTask"; + MapredWork mrWork = (MapredWork) currTask.getWork(); // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix Index: ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (working copy) @@ -216,16 +216,27 @@ public int execute(DriverContext driverContext) { LOG.info("Executing stats task"); - // Make sure that it is either an ANALYZE command or an INSERT OVERWRITE command - assert (work.getLoadTableDesc() != null && work.getTableSpecs() == null || work - .getLoadTableDesc() == null && work.getTableSpecs() != null); + // Make sure that it is either an ANALYZE, INSERT OVERWRITE or CTAS command + short workComponentsPresent = 0; + if (work.getLoadTableDesc() != null) + workComponentsPresent++; + if (work.getTableSpecs() != null) + workComponentsPresent++; + if (work.getLoadFileDesc() != null) + workComponentsPresent++; + + assert (workComponentsPresent == 1); + String tableName = ""; try { if (work.getLoadTableDesc() != null) { tableName = work.getLoadTableDesc().getTable().getTableName(); + } else if (work.getTableSpecs() != null){ + tableName = work.getTableSpecs().tableName; } else { - tableName = work.getTableSpecs().tableName; + tableName = work.getLoadFileDesc().getDestinationCreateTable(); } + table = db.getTable(tableName); } catch (HiveException e) { @@ -292,7 +303,7 @@ List partitions = getPartitionsList(); boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); - if (partitions == null) { + if (partitions == null || partitions.size() == 0) { // non-partitioned tables: if (!tableStatsExist && atomic) { return 0; @@ -310,7 +321,7 @@ // In case of a non-partitioned table, the key for stats temporary store is "rootDir" if (statsAggregator != null) { - updateStats(collectableStats, tblStats, statsAggregator, parameters, + updateStats(collectableStats, tblStats, statsAggregator, parameters, work.getAggKey(), atomic); statsAggregator.cleanUp(work.getAggKey()); } @@ -349,7 +360,7 @@ LOG.info("Stats aggregator : " + partitionID); if (statsAggregator != null) { - updateStats(collectableStats, newPartStats, statsAggregator, + updateStats(collectableStats, newPartStats, statsAggregator, parameters, partitionID, atomic); } else { for (String statType : collectableStats) { @@ -447,7 +458,7 @@ if (value != null) { longValue = Long.parseLong(value); - if (work.getLoadTableDesc() != null && + if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null) { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (working copy) @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,6 +34,7 @@ @Explain(displayName = "Create Table") public class CreateTableDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 1L; + String databaseName; String tableName; boolean isExternal; ArrayList cols; @@ -60,6 +60,26 @@ public CreateTableDesc() { } + public CreateTableDesc(String databaseName, String tableName, boolean isExternal, + List cols, List partCols, + List bucketCols, List sortCols, int numBuckets, + String fieldDelim, String fieldEscape, String collItemDelim, + String mapKeyDelim, String lineDelim, String comment, String inputFormat, + String outputFormat, String location, String serName, + String storageHandler, + Map serdeProps, + Map tblProps, + boolean ifNotExists) { + + this(tableName, isExternal, cols, partCols, + bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape, + collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat, + outputFormat, location, serName, storageHandler, serdeProps, + tblProps, ifNotExists); + + this.databaseName = databaseName; + } + public CreateTableDesc(String tableName, boolean isExternal, List cols, List partCols, List bucketCols, List sortCols, int numBuckets, @@ -117,6 +137,10 @@ return tableName; } + public String getDatabaseName(){ + return databaseName; + } + public void setTableName(String tableName) { this.tableName = tableName; } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (working copy) @@ -32,8 +32,9 @@ private tableSpec tableSpecs; // source table spec -- for TableScanOperator private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator + private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator private String aggKey; // aggregation key prefix - + private boolean noStatsAggregator = false; public StatsWork() { @@ -47,6 +48,10 @@ this.loadTableDesc = loadTableDesc; } + public StatsWork(LoadFileDesc loadFileDesc) { + this.loadFileDesc = loadFileDesc; + } + public tableSpec getTableSpecs() { return tableSpecs; } @@ -55,6 +60,10 @@ return loadTableDesc; } + public LoadFileDesc getLoadFileDesc() { + return loadFileDesc; + } + public void setAggKey(String aggK) { aggKey = aggK; } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -263,6 +263,11 @@ properties.setProperty(Constants.LINE_DELIM, crtTblDesc.getLineDelim()); } + if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, + crtTblDesc.getDatabaseName() + "." + crtTblDesc.getTableName()); + } + // replace the default input & output file format with those found in // crtTblDesc Class c1 = Class.forName(crtTblDesc.getInputFormat()); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java (working copy) @@ -31,10 +31,19 @@ // list of columns, comma separated private String columns; private String columnTypes; + private String destinationCreateTable; public LoadFileDesc() { } + public LoadFileDesc(final CreateTableDesc createTableDesc, final String sourceDir, final String targetDir, + final boolean isDfsDir, final String columns, final String columnTypes) { + this(sourceDir, targetDir, isDfsDir, columns, columnTypes); + if (createTableDesc != null && createTableDesc.getDatabaseName() != null && createTableDesc.getTableName() != null){ + destinationCreateTable = createTableDesc.getDatabaseName() + "." + createTableDesc.getTableName(); + } + } + public LoadFileDesc(final String sourceDir, final String targetDir, final boolean isDfsDir, final String columns, final String columnTypes) { @@ -92,4 +101,11 @@ public void setColumnTypes(String columnTypes) { this.columnTypes = columnTypes; } + + /** + * @return the destinationCreateTable + */ + public String getDestinationCreateTable(){ + return destinationCreateTable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -619,7 +619,8 @@ assert (ast.getToken().getType() == HiveParser.TOK_TAB || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION - || ast.getToken().getType() == HiveParser.TOK_TABTYPE); + || ast.getToken().getType() == HiveParser.TOK_TABTYPE + || ast.getToken().getType() == HiveParser.TOK_CREATETABLE); int childIndex = 0; numDynParts = 0; @@ -631,8 +632,9 @@ tableName = conf.getVar(HiveConf.ConfVars.HIVETESTMODEPREFIX) + tableName; } - - tableHandle = db.getTable(tableName); + if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + tableHandle = db.getTable(tableName); + } } catch (InvalidTableException ite) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(ast .getChild(0)), ite); @@ -642,7 +644,7 @@ } // get partition metadata if partition specified - if (ast.getChildCount() == 2) { + if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { childIndex = 1; ASTNode partspec = (ASTNode) ast.getChild(1); partitions = new ArrayList(); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1196269) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -64,6 +64,7 @@ import org.apache.hadoop.hive.ql.exec.RecordWriter; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -107,6 +108,10 @@ import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeCreateCommonVars; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.RowFormatParams; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.StorageFormat; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; @@ -1047,6 +1052,13 @@ throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location), e); } + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + tableSpec ts = new tableSpec(db, conf, this.ast); + // Set that variable to automatically collect stats during the MapReduce job + qb.getParseInfo().setIsInsertToTable(true); + // Add the table spec for the destination table. + qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); + } } else { qb.setIsQuery(true); fname = ctx.getMRTmpFileURI(); @@ -3990,7 +4002,7 @@ } boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE); - loadFileWork.add(new LoadFileDesc(queryTmpdir, destStr, isDfsDir, cols, + loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, destStr, isDfsDir, cols, colTypes)); if (tblDesc == null) { @@ -6916,6 +6928,10 @@ Path targetPath; try { dumpTable = db.newTable(qb.getTableDesc().getTableName()); + if (!db.databaseExists(dumpTable.getDbName())) { + throw new SemanticException("ERROR: The database " + dumpTable.getDbName() + + " does not exist."); + } Warehouse wh = new Warehouse(conf); targetPath = wh.getTablePath(db.getDatabase(dumpTable.getDbName()), dumpTable .getTableName()); @@ -7030,7 +7046,15 @@ getLeafTasks(rootTasks, leaves); assert (leaves.size() > 0); for (Task task : leaves) { - task.addDependentTask(crtTblTask); + if (task instanceof StatsTask){ + //StatsTask require table to already exist + for (Task parentOfStatsTask : task.getParentTasks()){ + parentOfStatsTask.addDependentTask(crtTblTask); + } + crtTblTask.addDependentTask(task); + } else { + task.addDependentTask(crtTblTask); + } } } @@ -7838,10 +7862,13 @@ case CTAS: // create table as select // Verify that the table does not already exist + String databaseName; try { Table dumpTable = db.newTable(tableName); + databaseName = dumpTable.getDbName(); if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false)) { - throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(tableName)); + //throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(tableName)); + ; } } catch (HiveException e) { throw new SemanticException(e); @@ -7849,7 +7876,7 @@ tblProps = addDefaultProperties(tblProps); - crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols, + crtTblDesc = new CreateTableDesc(databaseName, tableName, isExt, cols, partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, storageFormat.storageHandler, shared.serdeProps,