diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 1655f3d..77dd6b9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -1842,6 +1842,14 @@ private int showPartitions(Hive db, ShowPartitionsDesc showParts) throws HiveExc return 0; } + private static final String[] DELIMITER_PREFIXES = new String[] { + "FIELDS TERMINATED BY", + "COLLECTION ITEMS TERMINATED BY", + "MAP KEYS TERMINATED BY", + "LINES TERMINATED BY", + "NULL DEFINED AS" + }; + /** * Write a statement of how to create a table to a file. * @@ -1983,80 +1991,65 @@ else if (sortCol.getOrder() == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_DESC) { } // Row format (SerDe) - String tbl_row_format = ""; + StringBuilder tbl_row_format = new StringBuilder(); StorageDescriptor sd = tbl.getTTable().getSd(); SerDeInfo serdeInfo = sd.getSerdeInfo(); - tbl_row_format += "ROW FORMAT"; + tbl_row_format.append("ROW FORMAT"); if (tbl.getStorageHandler() == null) { - if (serdeInfo.getParametersSize() > 1) { + Map serdeParams = serdeInfo.getParameters(); + String[] delimiters = new String[] { + serdeParams.remove(serdeConstants.FIELD_DELIM), + serdeParams.remove(serdeConstants.COLLECTION_DELIM), + serdeParams.remove(serdeConstants.MAPKEY_DELIM), + serdeParams.remove(serdeConstants.LINE_DELIM), + serdeParams.remove(serdeConstants.SERIALIZATION_NULL_FORMAT) + }; + serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT); + if (containsNonNull(delimiters)) { // There is a "serialization.format" property by default, // even with a delimited row format. // But our result will only cover the following four delimiters. - tbl_row_format += " DELIMITED \n"; - Map delims = serdeInfo.getParameters(); + tbl_row_format.append(" DELIMITED \n"); + // Warn: // If the four delimiters all exist in a CREATE TABLE query, // this following order needs to be strictly followed, // or the query will fail with a ParseException. - if (delims.containsKey(serdeConstants.FIELD_DELIM)) { - tbl_row_format += " FIELDS TERMINATED BY '" + - escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get( - serdeConstants.FIELD_DELIM))) + "' \n"; - } - if (delims.containsKey(serdeConstants.COLLECTION_DELIM)) { - tbl_row_format += " COLLECTION ITEMS TERMINATED BY '" + - escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get( - serdeConstants.COLLECTION_DELIM))) + "' \n"; - } - if (delims.containsKey(serdeConstants.MAPKEY_DELIM)) { - tbl_row_format += " MAP KEYS TERMINATED BY '" + - escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get( - serdeConstants.MAPKEY_DELIM))) + "' \n"; - } - if (delims.containsKey(serdeConstants.LINE_DELIM)) { - tbl_row_format += " LINES TERMINATED BY '" + - escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get( - serdeConstants.LINE_DELIM))) + "' \n"; - } - if (delims.containsKey(serdeConstants.SERIALIZATION_NULL_FORMAT)) { - tbl_row_format += " NULL DEFINED AS '" + - escapeHiveCommand(StringEscapeUtils.escapeJava(delims.get( - serdeConstants.SERIALIZATION_NULL_FORMAT))) + "' \n"; + for (int i = 0; i < DELIMITER_PREFIXES.length; i++) { + if (delimiters[i] != null) { + tbl_row_format.append(" ").append(DELIMITER_PREFIXES[i]).append(" '"); + tbl_row_format.append(escapeHiveCommand(StringEscapeUtils.escapeJava(delimiters[i]))); + tbl_row_format.append("' \n"); + } } + } else { + tbl_row_format.append(" SERDE \n '" + + escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n"); } - else { - tbl_row_format += " SERDE \n '" + - escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n"; + if (!serdeParams.isEmpty()) { + appendSerdeParams(tbl_row_format, serdeParams).append(" \n"); } - tbl_row_format += "STORED AS INPUTFORMAT \n '" + - escapeHiveCommand(sd.getInputFormat()) + "' \n"; - tbl_row_format += "OUTPUTFORMAT \n '" + - escapeHiveCommand(sd.getOutputFormat()) + "'"; - } - else { + tbl_row_format.append("STORED AS INPUTFORMAT \n '" + + escapeHiveCommand(sd.getInputFormat()) + "' \n"); + tbl_row_format.append("OUTPUTFORMAT \n '" + + escapeHiveCommand(sd.getOutputFormat()) + "'"); + } else { duplicateProps.add(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); - tbl_row_format += " SERDE \n '" + - escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n"; - tbl_row_format += "STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)) + "' \n"; + tbl_row_format.append(" SERDE \n '" + + escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n"); + tbl_row_format.append("STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)) + "' \n"); // SerDe Properties if (serdeInfo.getParametersSize() > 0) { - tbl_row_format += "WITH SERDEPROPERTIES ( \n"; - List serdeCols = new ArrayList(); - for (Map.Entry entry : serdeInfo.getParameters().entrySet()) { - serdeCols.add(" '" + entry.getKey() + "'='" - + escapeHiveCommand(StringEscapeUtils.escapeJava(entry.getValue())) + "'"); - } - tbl_row_format += StringUtils.join(serdeCols, ", \n"); - tbl_row_format += ")"; + appendSerdeParams(tbl_row_format, serdeInfo.getParameters()); } } String tbl_location = " '" + escapeHiveCommand(sd.getLocation()) + "'"; // Table properties String tbl_properties = ""; - Map properties = new TreeMap(tbl.getParameters()); - if (properties.size() > 0) { + if (!tbl.getParameters().isEmpty()) { + Map properties = new TreeMap(tbl.getParameters()); List realProps = new ArrayList(); for (String key : properties.keySet()) { if (properties.get(key) != null && !duplicateProps.contains(key)) { @@ -2098,6 +2091,27 @@ else if (sortCol.getOrder() == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_DESC) { return 0; } + private boolean containsNonNull(String[] values) { + for (String value : values) { + if (value != null) { + return true; + } + } + return false; + } + + private StringBuilder appendSerdeParams(StringBuilder builder, Map serdeParam) { + serdeParam = new TreeMap(serdeParam); + builder.append("WITH SERDEPROPERTIES ( \n"); + List serdeCols = new ArrayList(); + for (Entry entry : serdeParam.entrySet()) { + serdeCols.add(" '" + entry.getKey() + "'='" + + escapeHiveCommand(StringEscapeUtils.escapeJava(entry.getValue())) + "'"); + } + builder.append(StringUtils.join(serdeCols, ", \n")).append(')'); + return builder; + } + /** * Write a list of indexes to a file. * diff --git ql/src/test/queries/clientpositive/show_create_table_serde.q ql/src/test/queries/clientpositive/show_create_table_serde.q index a3eb5a8..a94379b 100644 --- ql/src/test/queries/clientpositive/show_create_table_serde.q +++ ql/src/test/queries/clientpositive/show_create_table_serde.q @@ -1,5 +1,10 @@ -- Test SHOW CREATE TABLE on a table with serde. +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint); +ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2'); +SHOW CREATE TABLE tmp_showcrt1; +DROP TABLE tmp_showcrt1; + -- without a storage handler CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) COMMENT 'temporary table' @@ -9,6 +14,15 @@ OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'; SHOW CREATE TABLE tmp_showcrt1; DROP TABLE tmp_showcrt1; +-- without a storage handler / with custom serde params +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2') +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'; +SHOW CREATE TABLE tmp_showcrt1; +DROP TABLE tmp_showcrt1; + -- with a storage handler and serde properties CREATE EXTERNAL TABLE tmp_showcrt1 (key string, value boolean) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' diff --git ql/src/test/results/clientpositive/show_create_table_serde.q.out ql/src/test/results/clientpositive/show_create_table_serde.q.out index f1656b6..a794bcc 100644 --- ql/src/test/results/clientpositive/show_create_table_serde.q.out +++ ql/src/test/results/clientpositive/show_create_table_serde.q.out @@ -1,6 +1,61 @@ PREHOOK: query: -- Test SHOW CREATE TABLE on a table with serde. --- without a storage handler +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_showcrt1 +POSTHOOK: query: -- Test SHOW CREATE TABLE on a table with serde. + +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2') +PREHOOK: type: ALTERTABLE_SERDEPROPERTIES +PREHOOK: Input: default@tmp_showcrt1 +PREHOOK: Output: default@tmp_showcrt1 +POSTHOOK: query: ALTER TABLE tmp_showcrt1 SET SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2') +POSTHOOK: type: ALTERTABLE_SERDEPROPERTIES +POSTHOOK: Input: default@tmp_showcrt1 +POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: SHOW CREATE TABLE tmp_showcrt1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@tmp_showcrt1 +POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@tmp_showcrt1 +CREATE TABLE `tmp_showcrt1`( + `key` int, + `value` string, + `newvalue` bigint) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'custom.property.key1'='custom.property.value1', + 'custom.property.key2'='custom.property.value2') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'COLUMN_STATS_ACCURATE'='false', +#### A masked pattern was here #### + 'numFiles'='0', + 'numRows'='-1', + 'rawDataSize'='-1', + 'totalSize'='0', +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE tmp_showcrt1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmp_showcrt1 +PREHOOK: Output: default@tmp_showcrt1 +POSTHOOK: query: DROP TABLE tmp_showcrt1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmp_showcrt1 +POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: -- without a storage handler CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) COMMENT 'temporary table' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' @@ -9,9 +64,7 @@ OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@tmp_showcrt1 -POSTHOOK: query: -- Test SHOW CREATE TABLE on a table with serde. - --- without a storage handler +POSTHOOK: query: -- without a storage handler CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) COMMENT 'temporary table' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' @@ -49,6 +102,55 @@ POSTHOOK: query: DROP TABLE tmp_showcrt1 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tmp_showcrt1 POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: -- without a storage handler / with custom serde params +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2') +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp_showcrt1 +POSTHOOK: query: -- without a storage handler / with custom serde params +CREATE TABLE tmp_showcrt1 (key int, value string, newvalue bigint) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +WITH SERDEPROPERTIES ('custom.property.key1'='custom.property.value1', 'custom.property.key2'='custom.property.value2') +STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: SHOW CREATE TABLE tmp_showcrt1 +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@tmp_showcrt1 +POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1 +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@tmp_showcrt1 +CREATE TABLE `tmp_showcrt1`( + `key` int, + `value` string, + `newvalue` bigint) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +WITH SERDEPROPERTIES ( + 'custom.property.key1'='custom.property.value1', + 'custom.property.key2'='custom.property.value2') +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE tmp_showcrt1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmp_showcrt1 +PREHOOK: Output: default@tmp_showcrt1 +POSTHOOK: query: DROP TABLE tmp_showcrt1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmp_showcrt1 +POSTHOOK: Output: default@tmp_showcrt1 PREHOOK: query: -- with a storage handler and serde properties CREATE EXTERNAL TABLE tmp_showcrt1 (key string, value boolean) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' @@ -79,8 +181,8 @@ ROW FORMAT SERDE STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' WITH SERDEPROPERTIES ( - 'serialization.format'='$', - 'field.delim'=',') + 'field.delim'=',', + 'serialization.format'='$') LOCATION #### A masked pattern was here #### TBLPROPERTIES (