diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 8a06582..7d2595d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2094,7 +2094,11 @@ private void getMetaData(QB qb, ReadEntity parentInput) ASTNode serdeChild = (ASTNode) child.getChild(0); storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); directoryDesc.setSerName(storageFormat.getSerde()); - directoryDescIsSet=true; + if (serdeChild.getChildCount() > 1) { + directoryDesc.setSerdeProps(new HashMap()); + readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps()); + } + directoryDescIsSet = true; break; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 04d26f3..990a10c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -141,6 +141,9 @@ public static TableDesc getDefaultTableDesc(CreateTableDesc directoryDesc, properties.setProperty( serdeConstants.SERIALIZATION_LIB, directoryDesc.getSerName()); } + if (directoryDesc.getSerdeProps() != null) { + properties.putAll(directoryDesc.getSerdeProps()); + } if (directoryDesc.getOutputFormat() != null){ ret.setOutputFileFormatClass(JavaUtils.loadClass(directoryDesc.getOutputFormat())); } diff --git a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q index c3d419b..2775314 100644 --- a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q +++ b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q @@ -80,8 +80,8 @@ insert overwrite directory '../../data/files/array_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=',', +'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE select a, null, b from array_table; dfs -cat ../../data/files/array_table_4/000000_0; @@ -97,8 +97,8 @@ insert overwrite directory '../../data/files/map_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=':', +'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS TEXTFILE select foo, null, bar from map_table; dfs -cat ../../data/files/map_table_4/000000_0; diff --git a/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out b/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out index e5d2455..a459839 100644 --- a/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out +++ b/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out @@ -1190,8 +1190,8 @@ PREHOOK: query: insert overwrite directory '../../data/files/array_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=',', +'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE select a, null, b from array_table PREHOOK: type: QUERY PREHOOK: Input: default@array_table @@ -1200,14 +1200,14 @@ POSTHOOK: query: insert overwrite directory '../../data/files/array_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=',', +'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE select a, null, b from array_table POSTHOOK: type: QUERY POSTHOOK: Input: default@array_table #### A masked pattern was here #### -a1a2a3\Nb1b2b3b4 -a21a22a23\Nb21b22b23b24 +a1#a2#a3,-NA-,b1#b2#b3#b4 +a21#a22#a23,-NA-,b21#b22#b23#b24 PREHOOK: query: insert overwrite directory '../../data/files/map_table_3' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe' STORED AS TEXTFILE @@ -1228,8 +1228,8 @@ PREHOOK: query: insert overwrite directory '../../data/files/map_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=':', +'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS TEXTFILE select foo, null, bar from map_table PREHOOK: type: QUERY PREHOOK: Input: default@map_table @@ -1238,14 +1238,14 @@ POSTHOOK: query: insert overwrite directory '../../data/files/map_table_4' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol', -'quote.delim'= '("|\\[|\\])', 'field.delim'=', ', -'serialization.null.format'='-' ) STORED AS TEXTFILE +'quote.delim'= '("|\\[|\\])', 'field.delim'=':', +'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS TEXTFILE select foo, null, bar from map_table POSTHOOK: type: QUERY POSTHOOK: Input: default@map_table #### A masked pattern was here #### -foo1\Nk1v1k2v2k3v3 -foo2\Nk21v21k22v22k31v31 +foo1:-NA-:k1%v1#k2%v2#k3%v3 +foo2:-NA-:k21%v21#k22%v22#k31%v31 PREHOOK: query: insert overwrite directory '../../data/files/rctable' STORED AS RCFILE select value,key from src