Index: ql/src/test/results/clientpositive/transform_ppr2.q.out =================================================================== --- ql/src/test/results/clientpositive/transform_ppr2.q.out (revision 825659) +++ ql/src/test/results/clientpositive/transform_ppr2.q.out (working copy) @@ -57,6 +57,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + field.delim 9 columns ds,tkey,tvalue serialization.format 9 columns.types string,string,string @@ -78,10 +79,10 @@ type: string Needs Tagging: false Path -> Alias: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [tmap:src] - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [tmap:src] Path -> Partition: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition partition values: ds 2008-04-08 @@ -100,10 +101,11 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255672850 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -122,7 +124,8 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255672850 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart Reduce Operator Tree: @@ -142,7 +145,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/data/users/njain/hive5/hive5/build/ql/tmp/1848215101/10001 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1839252793/10001 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -167,7 +170,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/1568626028/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1775717975/10000 POSTHOOK: query: FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -179,7 +182,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/1568626028/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1775717975/10000 0 val_0 0 val_0 0 val_0 Index: ql/src/test/results/clientpositive/ctas.q.out =================================================================== --- ql/src/test/results/clientpositive/ctas.q.out (revision 825659) +++ ql/src/test/results/clientpositive/ctas.q.out (working copy) @@ -10,6 +10,14 @@ PREHOOK: type: DROPTABLE POSTHOOK: query: drop table nzhang_ctas3 POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table nzhang_ctas4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_ctas4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table nzhang_ctas5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_ctas5 +POSTHOOK: type: DROPTABLE PREHOOK: query: explain create table nzhang_ctas1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE POSTHOOK: query: explain create table nzhang_ctas1 as select key k, value from src sort by k, value limit 10 @@ -63,7 +71,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1066079488/10002 + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/551361482/10002 Reduce Output Operator key expressions: expr: _col0 @@ -91,7 +99,7 @@ Move Operator files: hdfs directory: true - destination: file:///data/users/nzhang/work/31/apache-hive-trunk/ql/../build/ql/test/data/warehouse//nzhang_ctas1 + destination: file:///data/users/nzhang/work/876/apache-hive/ql/../build/ql/test/data/warehouse/nzhang_ctas1 Stage: Stage-3 Create Table Operator: @@ -115,11 +123,11 @@ PREHOOK: query: select * from nzhang_ctas1 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas1 -PREHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1419754075/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/170193651/10000 POSTHOOK: query: select * from nzhang_ctas1 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas1 -POSTHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1419754075/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/170193651/10000 0 val_0 0 val_0 0 val_0 @@ -183,7 +191,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/574523558/10002 + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1299285756/10002 Reduce Output Operator key expressions: expr: _col0 @@ -211,7 +219,7 @@ Move Operator files: hdfs directory: true - destination: file:///data/users/nzhang/work/31/apache-hive-trunk/ql/../build/ql/test/data/warehouse//nzhang_ctas2 + destination: file:///data/users/nzhang/work/876/apache-hive/ql/../build/ql/test/data/warehouse/nzhang_ctas2 Stage: Stage-3 Create Table Operator: @@ -235,11 +243,11 @@ PREHOOK: query: select * from nzhang_ctas2 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas2 -PREHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1229264510/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1593006253/10000 POSTHOOK: query: select * from nzhang_ctas2 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas2 -POSTHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1229264510/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1593006253/10000 0 val_0 0 val_0 0 val_0 @@ -303,7 +311,7 @@ Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1141672779/10002 + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/115382/10002 Reduce Output Operator key expressions: expr: _col0 @@ -331,7 +339,7 @@ Move Operator files: hdfs directory: true - destination: file:///data/users/nzhang/work/31/apache-hive-trunk/ql/../build/ql/test/data/warehouse//nzhang_ctas3 + destination: file:///data/users/nzhang/work/876/apache-hive/ql/../build/ql/test/data/warehouse/nzhang_ctas3 Stage: Stage-3 Create Table Operator: @@ -356,11 +364,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1336526193/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/263363884/10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1336526193/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/263363884/10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -389,11 +397,11 @@ PREHOOK: query: select * from nzhang_ctas3 PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_ctas3 -PREHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1592752406/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/103710196/10000 POSTHOOK: query: select * from nzhang_ctas3 POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_ctas3 -POSTHOOK: Output: file:/data/users/nzhang/work/31/apache-hive-trunk/build/ql/tmp/1592752406/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/103710196/10000 0.0 val_0_con 0.0 val_0_con 0.0 val_0_con @@ -404,6 +412,280 @@ 2.5 val_5_con 4.0 val_8_con 4.5 val_9_con +PREHOOK: query: explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 +POSTHOOK: type: CREATETABLE +ABSTRACT SYNTAX TREE: + (TOK_CREATETABLE nzhang_ctas4 TOK_LIKETABLE (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD ','))) TOK_TBLTEXTFILE (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/473374758/10002 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true + destination: file:///data/users/nzhang/work/876/apache-hive/ql/../build/ql/test/data/warehouse/nzhang_ctas4 + + Stage: Stage-3 + Create Table Operator: + Create Table + columns: key string, value string + field delimiter: , + if not exists: false + input format: org.apache.hadoop.mapred.TextInputFormat + # buckets: -1 + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + name: nzhang_ctas4 + isExternal: false + + +PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 +PREHOOK: type: CREATETABLE +PREHOOK: Input: default@src +POSTHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_ctas4 +PREHOOK: query: select * from nzhang_ctas4 +PREHOOK: type: QUERY +PREHOOK: Input: default@nzhang_ctas4 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1232078865/10000 +POSTHOOK: query: select * from nzhang_ctas4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@nzhang_ctas4 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1232078865/10000 +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +PREHOOK: query: explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10 +PREHOOK: type: CREATETABLE +POSTHOOK: query: explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10 +POSTHOOK: type: CREATETABLE +ABSTRACT SYNTAX TREE: + (TOK_CREATETABLE nzhang_ctas5 TOK_LIKETABLE (TOK_TABLEROWFORMAT (TOK_SERDEPROPS (TOK_TABLEROWFORMATFIELD ',') (TOK_TABLEROWFORMATLINES '.'))) TOK_TBLTEXTFILE (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 10)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Needs Tagging: false + Path -> Alias: + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src [src] + Path -> Partition: + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src + Partition + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + name src + columns.types string:string + serialization.ddl struct src { string key, string value} + serialization.format 1 + columns key,value + bucket_count -1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src + transient_lastDdlTime 1255728382 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: src + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10002 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10002 + Reduce Output Operator + key expressions: + expr: _col0 + type: string + expr: _col1 + type: string + sort order: ++ + tag: -1 + value expressions: + expr: _col0 + type: string + expr: _col1 + type: string + Needs Tagging: false + Path -> Alias: + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10002 [file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10002] + Path -> Partition: + file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10002 + Partition + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types string,string + escape.delim \ + Reduce Operator Tree: + Extract + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10001 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + field.delim , + line.delim . + columns _col0,_col1 + serialization.format , + columns.types string:string + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true + source: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/751741251/10001 + destination: file:///data/users/nzhang/work/876/apache-hive/ql/../build/ql/test/data/warehouse/nzhang_ctas5 + + Stage: Stage-3 + Create Table Operator: + Create Table + columns: key string, value string + field delimiter: , + if not exists: false + input format: org.apache.hadoop.mapred.TextInputFormat + line delimiter: . + # buckets: -1 + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + name: nzhang_ctas5 + isExternal: false + + +PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10 +PREHOOK: type: CREATETABLE +PREHOOK: Input: default@src +POSTHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10 +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_ctas5 PREHOOK: query: drop table nzhang_ctas1 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table nzhang_ctas1 @@ -419,3 +701,13 @@ POSTHOOK: query: drop table nzhang_ctas3 POSTHOOK: type: DROPTABLE POSTHOOK: Output: default@nzhang_ctas3 +PREHOOK: query: drop table nzhang_ctas4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_ctas4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@nzhang_ctas4 +PREHOOK: query: drop table nzhang_ctas5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_ctas5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@nzhang_ctas5 Index: ql/src/test/results/clientpositive/transform_ppr1.q.out =================================================================== --- ql/src/test/results/clientpositive/transform_ppr1.q.out (revision 825659) +++ ql/src/test/results/clientpositive/transform_ppr1.q.out (working copy) @@ -45,6 +45,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + field.delim 9 columns ds,tkey,tvalue serialization.format 9 columns.types string,string,string @@ -66,12 +67,12 @@ type: string Needs Tagging: false Path -> Alias: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [tmap:src] - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [tmap:src] - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=11 [tmap:src] - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=11 [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 [tmap:src] Path -> Partition: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition partition values: ds 2008-04-08 @@ -90,10 +91,11 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255673097 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -112,10 +114,11 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255673097 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=11 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=11 Partition partition values: ds 2008-04-09 @@ -134,10 +137,11 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255673097 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 Partition partition values: ds 2008-04-09 @@ -156,7 +160,8 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/srcpart + transient_lastDdlTime 1255673097 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcpart Reduce Operator Tree: @@ -176,7 +181,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/data/users/njain/hive5/hive5/build/ql/tmp/1777811386/10001 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1989886167/10001 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -202,7 +207,7 @@ PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/357469194/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1814497565/10000 POSTHOOK: query: FROM ( FROM srcpart src SELECT TRANSFORM(src.ds, src.key, src.value) @@ -215,7 +220,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/357469194/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1814497565/10000 0 val_0 0 val_0 0 val_0 Index: ql/src/test/results/clientpositive/regexp_extract.q.out =================================================================== --- ql/src/test/results/clientpositive/regexp_extract.q.out (revision 825659) +++ ql/src/test/results/clientpositive/regexp_extract.q.out (working copy) @@ -47,6 +47,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + field.delim 9 columns key,value serialization.format 9 columns.types string,string @@ -67,9 +68,9 @@ type: string Needs Tagging: false Path -> Alias: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src [tmap:src] Path -> Partition: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src Partition input format: org.apache.hadoop.mapred.TextInputFormat @@ -84,7 +85,8 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src + transient_lastDdlTime 1255673006 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src Reduce Operator Tree: @@ -104,7 +106,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/data/users/njain/hive5/hive5/build/ql/tmp/1466313209/10001 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1866661400/10001 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -127,7 +129,7 @@ SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/718076594/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/951679285/10000 POSTHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) @@ -137,7 +139,7 @@ SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)',1) WHERE tmap.key < 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/718076594/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/951679285/10000 0 0 3 0 0 3 0 0 3 @@ -271,6 +273,7 @@ input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + field.delim 9 columns key,value serialization.format 9 columns.types string,string @@ -291,9 +294,9 @@ type: string Needs Tagging: false Path -> Alias: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src [tmap:src] + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src [tmap:src] Path -> Partition: - file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src Partition input format: org.apache.hadoop.mapred.TextInputFormat @@ -308,7 +311,8 @@ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/src + location file:/data/users/nzhang/work/876/apache-hive/build/ql/test/data/warehouse/src + transient_lastDdlTime 1255673006 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: src Reduce Operator Tree: @@ -328,7 +332,7 @@ File Output Operator compressed: false GlobalTableId: 0 - directory: file:/data/users/njain/hive5/hive5/build/ql/tmp/1577694586/10001 + directory: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1917765093/10001 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -351,7 +355,7 @@ SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 PREHOOK: type: QUERY PREHOOK: Input: default@src -PREHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/137072546/10000 +PREHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1107567965/10000 POSTHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value, 1+2, 3+4) @@ -361,7 +365,7 @@ SELECT tmap.key, regexp_extract(tmap.value, 'val_(\\d+\\t\\d+)') WHERE tmap.key < 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@src -POSTHOOK: Output: file:/data/users/njain/hive5/hive5/build/ql/tmp/137072546/10000 +POSTHOOK: Output: file:/data/users/nzhang/work/876/apache-hive/build/ql/tmp/1107567965/10000 0 0 3 0 0 3 0 0 3 Index: ql/src/test/results/compiler/plan/input4.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input4.q.xml (revision 825659) +++ ql/src/test/results/compiler/plan/input4.q.xml (working copy) @@ -26,7 +26,7 @@ true - file:/data/users/njain/hive3/hive3/build/ql/tmp/1522250334/10000 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/1849977570/10000 @@ -79,11 +79,11 @@ location - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/dest1 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/dest1 transient_lastDdlTime - 1254943979 + 1255671604 @@ -93,7 +93,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/tmp/1522250334/10001 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/1849977570/10001 @@ -166,11 +166,11 @@ location - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src transient_lastDdlTime - 1254943978 + 1255671604 @@ -432,6 +432,10 @@ + field.delim + 9 + + columns _col0,_col1 @@ -465,6 +469,10 @@ + field.delim + 9 + + columns tkey,tvalue @@ -701,7 +709,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src tmap:src @@ -713,7 +721,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src @@ -761,7 +769,7 @@ 1 - file:/data/users/njain/hive3/hive3/build/ql/tmp/1522250334/10000 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/1849977570/10000 Index: ql/src/test/results/compiler/plan/input5.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input5.q.xml (revision 825659) +++ ql/src/test/results/compiler/plan/input5.q.xml (working copy) @@ -26,7 +26,7 @@ true - file:/data/users/njain/hive3/hive3/build/ql/tmp/228410659/10000 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/702311970/10000 @@ -79,11 +79,11 @@ location - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/dest1 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/dest1 transient_lastDdlTime - 1254943981 + 1255671608 @@ -93,7 +93,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/tmp/228410659/10001 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/702311970/10001 @@ -170,11 +170,11 @@ location - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src_thrift + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src_thrift transient_lastDdlTime - 1254943980 + 1255671607 @@ -436,6 +436,10 @@ + field.delim + 9 + + columns _col0,_col1 @@ -469,6 +473,10 @@ + field.delim + 9 + + columns tkey,tvalue @@ -768,7 +776,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src_thrift + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src_thrift tmap:src_thrift @@ -780,7 +788,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src_thrift + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src_thrift @@ -824,7 +832,7 @@ 1 - file:/data/users/njain/hive3/hive3/build/ql/tmp/228410659/10000 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/702311970/10000 Index: ql/src/test/results/compiler/plan/input20.q.xml =================================================================== --- ql/src/test/results/compiler/plan/input20.q.xml (revision 825659) +++ ql/src/test/results/compiler/plan/input20.q.xml (working copy) @@ -64,11 +64,11 @@ location - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src transient_lastDdlTime - 1254943974 + 1255671598 @@ -330,6 +330,10 @@ + field.delim + 9 + + columns _col0,_col1 @@ -363,6 +367,10 @@ + field.delim + 9 + + columns key,value @@ -676,7 +684,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src tmap:src @@ -688,7 +696,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/src + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/test/data/warehouse/src @@ -733,7 +741,7 @@ - file:/data/users/njain/hive3/hive3/build/ql/tmp/1191035427/10001 + file:/data/users/nzhang/work/876/apache-hive/.ptest_3/build/ql/tmp/2104013641/10001 @@ -846,6 +854,10 @@ + field.delim + 9 + + columns _col0,_col1 @@ -879,6 +891,10 @@ + field.delim + 9 + + columns key,value Index: ql/src/test/queries/clientpositive/ctas.q =================================================================== --- ql/src/test/queries/clientpositive/ctas.q (revision 825659) +++ ql/src/test/queries/clientpositive/ctas.q (working copy) @@ -1,6 +1,8 @@ drop table nzhang_ctas1; drop table nzhang_ctas2; drop table nzhang_ctas3; +drop table nzhang_ctas4; +drop table nzhang_ctas5; explain create table nzhang_ctas1 as select key k, value from src sort by k, value limit 10; @@ -29,6 +31,19 @@ select * from nzhang_ctas3; + +explain create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10; + +create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10; + +select * from nzhang_ctas4; + +explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10; + +create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '.' stored as textfile as select key, value from src sort by key, value limit 10; + drop table nzhang_ctas1; drop table nzhang_ctas2; drop table nzhang_ctas3; +drop table nzhang_ctas4; +drop table nzhang_ctas5; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 825659) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -122,6 +122,10 @@ Properties properties = Utilities.makeProperties( Constants.SERIALIZATION_FORMAT, separatorCode, Constants.LIST_COLUMNS, columns); + + if ( ! separatorCode.equals(Integer.toString(Utilities.ctrlaCode)) ) + properties.setProperty(Constants.FIELD_DELIM, separatorCode); + if (columnTypes != null) properties.setProperty(Constants.LIST_COLUMN_TYPES, columnTypes); @@ -165,9 +169,27 @@ serdeClass = c; } + if ( crtTblDesc.getFieldDelim() != null ) + separatorCode = crtTblDesc.getFieldDelim(); + ret = getTableDesc(serdeClass, separatorCode, columns, columnTypes, - lastColumnTakesRestOfTheLine, false); + lastColumnTakesRestOfTheLine, false); + // set other table properties + Properties properties = ret.getProperties(); + + if ( crtTblDesc.getCollItemDelim() != null ) + properties.setProperty(Constants.COLLECTION_DELIM, crtTblDesc.getCollItemDelim()); + + if ( crtTblDesc.getMapKeyDelim() != null ) + properties.setProperty(Constants.MAPKEY_DELIM, crtTblDesc.getMapKeyDelim()); + + if ( crtTblDesc.getFieldEscape() != null ) + properties.setProperty(Constants.ESCAPE_CHAR, crtTblDesc.getFieldEscape()); + + if ( crtTblDesc.getLineDelim() != null ) + properties.setProperty(Constants.LINE_DELIM, crtTblDesc.getLineDelim()); + // replace the default input & output file format with those found in crtTblDesc Class c1 = Class.forName(crtTblDesc.getInputFormat()); Class c2 = Class.forName(crtTblDesc.getOutputFormat()); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 825659) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -1066,7 +1066,7 @@ if (rowChild.getChildCount()>=2) { String fieldEscape = unescapeSQLString(rowChild.getChild(1).getText()); - tblDesc.getProperties().setProperty(Constants.ESCAPE_CHAR, fieldDelim); + tblDesc.getProperties().setProperty(Constants.ESCAPE_CHAR, fieldEscape); } break; case HiveParser.TOK_TABLEROWFORMATCOLLITEMS: