From d0ec902ea43e4eb6a1ba235561569f0ab0c00767 Mon Sep 17 00:00:00 2001 From: nareshpr Date: Fri, 10 Apr 2020 17:19:24 -0700 Subject: [PATCH] HIVE-23179 - Show create table is not showing SerDe Properties in unicode --- .../hive/common/util/HiveStringUtils.java | 14 ++ .../create/show/ShowCreateTableOperation.java | 10 +- .../show_create_table_delimited.q | 15 ++ .../show_create_table_delimited.q.out | 156 ++++++++++++++++++ 4 files changed, 190 insertions(+), 5 deletions(-) diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java index 22948e38de..6499ac1639 100644 --- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java +++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java @@ -43,6 +43,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.text.translate.CharSequenceTranslator; import org.apache.commons.lang3.text.translate.EntityArrays; +import org.apache.commons.lang3.text.translate.JavaUnicodeEscaper; import org.apache.commons.lang3.text.translate.LookupTranslator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -83,6 +84,9 @@ }).with( new LookupTranslator(EntityArrays.JAVA_CTRL_CHARS_ESCAPE())); + private static final CharSequenceTranslator UNICODE_CONVERTER = + JavaUnicodeEscaper.outsideOf(32, 127); + static { NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH); decimalFormat = (DecimalFormat) numberFormat; @@ -651,6 +655,16 @@ public static String escapeHiveCommand(String str) { return ESCAPE_HIVE_COMMAND.translate(str); } + /** + * Escape java unicode characters. + * + * @param str Original string + * @return Escaped string + */ + public static String escapeUnicode(String str) { + return UNICODE_CONVERTER.translate(str); + } + /** * Unescape commas in the string using the default escape char * @param str a string diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java index 51d9f10436..bf913442f9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/show/ShowCreateTableOperation.java @@ -18,10 +18,6 @@ package org.apache.hadoop.hive.ql.ddl.table.create.show; -import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; -import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation; - import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; import java.io.DataOutputStream; @@ -47,6 +43,9 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.util.DirectionUtils; @@ -322,7 +321,8 @@ public static void appendSerdeParams(StringBuilder builder, Map SortedMap sortedSerdeParams = new TreeMap(serdeParams); List serdeCols = new ArrayList(); for (Entry entry : sortedSerdeParams.entrySet()) { - serdeCols.add(" '" + entry.getKey() + "'='" + HiveStringUtils.escapeHiveCommand(entry.getValue()) + "'"); + serdeCols.add(" '" + entry.getKey() + "'='" + + HiveStringUtils.escapeUnicode(HiveStringUtils.escapeHiveCommand(entry.getValue())) + "'"); } builder diff --git a/ql/src/test/queries/clientpositive/show_create_table_delimited.q b/ql/src/test/queries/clientpositive/show_create_table_delimited.q index 77229641f1..4eef9d52fa 100644 --- a/ql/src/test/queries/clientpositive/show_create_table_delimited.q +++ b/ql/src/test/queries/clientpositive/show_create_table_delimited.q @@ -7,3 +7,18 @@ LOCATION 'file:${system:test.tmp.dir}/tmp_showcrt1'; SHOW CREATE TABLE tmp_showcrt1; DROP TABLE tmp_showcrt1; +create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile; +show create table esc_special_delimiter; +drop table esc_special_delimiter; + +create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile; +show create table esc_special_delimiter; +drop table esc_special_delimiter; + +create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile; +show create table esc_special_delimiter; +drop table esc_special_delimiter; + +create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile; +show create table esc_special_delimiter; +drop table esc_special_delimiter; diff --git a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out index ad58bf62d3..fb88edb7d8 100644 --- a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out +++ b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out @@ -49,3 +49,159 @@ POSTHOOK: query: DROP TABLE tmp_showcrt1 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@tmp_showcrt1 POSTHOOK: Output: default@tmp_showcrt1 +PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\u0001' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: show create table esc_special_delimiter +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@esc_special_delimiter +POSTHOOK: query: show create table esc_special_delimiter +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@esc_special_delimiter +CREATE TABLE `esc_special_delimiter`( + `age` int, + `name` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='\u0001', + 'serialization.format'='\u0001') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: drop table esc_special_delimiter +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@esc_special_delimiter +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: drop table esc_special_delimiter +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@esc_special_delimiter +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '`' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: show create table esc_special_delimiter +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@esc_special_delimiter +POSTHOOK: query: show create table esc_special_delimiter +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@esc_special_delimiter +CREATE TABLE `esc_special_delimiter`( + `age` int, + `name` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='`', + 'serialization.format'='`') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: drop table esc_special_delimiter +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@esc_special_delimiter +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: drop table esc_special_delimiter +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@esc_special_delimiter +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\'' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: show create table esc_special_delimiter +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@esc_special_delimiter +POSTHOOK: query: show create table esc_special_delimiter +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@esc_special_delimiter +CREATE TABLE `esc_special_delimiter`( + `age` int, + `name` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='\'', + 'serialization.format'='\'') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: drop table esc_special_delimiter +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@esc_special_delimiter +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: drop table esc_special_delimiter +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@esc_special_delimiter +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: create table esc_special_delimiter(age int, name string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@esc_special_delimiter +PREHOOK: query: show create table esc_special_delimiter +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@esc_special_delimiter +POSTHOOK: query: show create table esc_special_delimiter +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@esc_special_delimiter +CREATE TABLE `esc_special_delimiter`( + `age` int, + `name` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +WITH SERDEPROPERTIES ( + 'field.delim'='\;', + 'serialization.format'='\;') +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: drop table esc_special_delimiter +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@esc_special_delimiter +PREHOOK: Output: default@esc_special_delimiter +POSTHOOK: query: drop table esc_special_delimiter +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@esc_special_delimiter +POSTHOOK: Output: default@esc_special_delimiter -- 2.20.1