diff --git metastore/scripts/upgrade/mysql/015-HIVE-5962.mysql.sql metastore/scripts/upgrade/mysql/015-HIVE-5962.mysql.sql new file mode 100644 index 0000000..4e903c7 --- /dev/null +++ metastore/scripts/upgrade/mysql/015-HIVE-5962.mysql.sql @@ -0,0 +1,10 @@ +-- +-- Hive HIVE-5962 +-- Support UTF-8 + +ALTER TABLE `SERDE_PARAMS` MODIFY PARAM_KEY VARCHAR(255); +ALTER TABLE `INDEX_PARAMS` MODIFY PARAM_KEY VARCHAR(255); +ALTER TABLE `PARTITION_PARAMS` MODIFY PARAM_KEY VARCHAR(255); +ALTER TABLE `TABLE_PARAMS` MODIFY PARAM_KEY VARCHAR(255); +ALTER TABLE `SD_PARAMS` MODIFY PARAM_KEY VARCHAR(255); +ALTER TABLE `DELEGATION_TOKENS` MODIFY TOKEN_IDENT VARCHAR(255); diff --git metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql index 2cd8db8..8f6ad8d 100644 --- metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql +++ metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql @@ -171,7 +171,7 @@ CREATE TABLE IF NOT EXISTS `IDXS` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( `INDEX_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_KEY` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`), KEY `INDEX_PARAMS_N49` (`INDEX_ID`), @@ -281,7 +281,7 @@ CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( `PART_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_KEY` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, PRIMARY KEY (`PART_ID`,`PARAM_KEY`), KEY `PARTITION_PARAMS_N49` (`PART_ID`), @@ -406,7 +406,7 @@ CREATE TABLE IF NOT EXISTS `SDS` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( `SD_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_KEY` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, PRIMARY KEY (`SD_ID`,`PARAM_KEY`), KEY `SD_PARAMS_N49` (`SD_ID`), @@ -449,7 +449,7 @@ CREATE TABLE IF NOT EXISTS `SERDES` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( `SERDE_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_KEY` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, PRIMARY KEY (`SERDE_ID`,`PARAM_KEY`), KEY `SERDE_PARAMS_N49` (`SERDE_ID`), @@ -562,7 +562,7 @@ CREATE TABLE IF NOT EXISTS `SORT_COLS` ( /*!40101 SET character_set_client = utf8 */; CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( `TBL_ID` bigint(20) NOT NULL, - `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_KEY` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, PRIMARY KEY (`TBL_ID`,`PARAM_KEY`), KEY `TABLE_PARAMS_N49` (`TBL_ID`), @@ -746,7 +746,7 @@ CREATE TABLE IF NOT EXISTS `MASTER_KEYS` -- Table `DELEGATION_TOKENS` for classes [org.apache.hadoop.hive.metastore.model.MDelegationToken] CREATE TABLE IF NOT EXISTS `DELEGATION_TOKENS` ( - `TOKEN_IDENT` VARCHAR(767) BINARY NOT NULL, + `TOKEN_IDENT` VARCHAR(255) BINARY NOT NULL, `TOKEN` VARCHAR(767) BINARY NULL, PRIMARY KEY (`TOKEN_IDENT`) ) ENGINE=INNODB DEFAULT CHARSET=latin1; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index d0e94bf..a8e5af1 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -500,7 +500,7 @@ static public void deleteWHDirectory(Path path, Configuration conf, * if it doesn't match the pattern. */ static public boolean validateName(String name) { - Pattern tpat = Pattern.compile("[\\w_]+"); + Pattern tpat = Pattern.compile("[\\w_가-힣]+"); Matcher m = tpat.matcher(name); if (m.matches()) { return true; diff --git metastore/src/model/package.jdo metastore/src/model/package.jdo index 49f2aac..0cf2863 100644 --- metastore/src/model/package.jdo +++ metastore/src/model/package.jdo @@ -161,7 +161,7 @@ - + @@ -194,7 +194,7 @@ - + @@ -309,7 +309,7 @@ - + @@ -373,7 +373,7 @@ - + @@ -420,7 +420,7 @@ - + @@ -778,7 +778,7 @@ - + diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 366b714..3911a36 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -308,6 +308,11 @@ Letter ; fragment +UTF + : ('\uAC00'..'\uD7AF') + ; + +fragment HexDigit : 'a'..'f' | 'A'..'F' ; @@ -377,7 +382,7 @@ Number Identifier : - (Letter | Digit) (Letter | Digit | '_')* + (Letter | UTF | Digit) (Letter | UTF | Digit | '_')* | '`' RegexComponent+ '`' ;