diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 571c789edd..4f88ef7f52 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -645,15 +645,14 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw // Get most of the fields for the IDs provided. // Assume db and table names are the same for all partition, as provided in arguments. String queryText = - "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," - + " " + SERDES + ".\"SERDE_ID\", " + PARTITIONS + ".\"CREATE_TIME\"," - + " " + PARTITIONS + ".\"LAST_ACCESS_TIME\", " + SDS + ".\"INPUT_FORMAT\", " + SDS + ".\"IS_COMPRESSED\"," - + " " + SDS + ".\"IS_STOREDASSUBDIRECTORIES\", " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," - + " " + SDS + ".\"OUTPUT_FORMAT\", " + SERDES + ".\"NAME\", " + SERDES + ".\"SLIB\", " + PARTITIONS - + ".\"WRITE_ID\"" + " from " + PARTITIONS + "" - + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " - + " left outer join " + SERDES + " on " + SDS + ".\"SERDE_ID\" = " + SERDES + ".\"SERDE_ID\" " - + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; + "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," + " " + SERDES + + ".\"SERDE_ID\", " + PARTITIONS + ".\"CREATE_TIME\"," + " " + PARTITIONS + ".\"LAST_ACCESS_TIME\", " + SDS + + ".\"INPUT_FORMAT\", " + SDS + ".\"IS_COMPRESSED\"," + " " + SDS + ".\"IS_STOREDASSUBDIRECTORIES\", " + SDS + + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + " " + SDS + ".\"OUTPUT_FORMAT\", " + SERDES + + ".\"NAME\", " + SERDES + ".\"SLIB\", " + PARTITIONS + ".\"WRITE_ID\"," + SERDES + ".\"SLIB_INT\"" + + " from " + PARTITIONS + "" + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + + ".\"SD_ID\" " + " left outer join " + SERDES + " on " + SDS + ".\"SERDE_ID\" = " + SERDES + + ".\"SERDE_ID\" " + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); List sqlResult = executeWithArray(query, null, queryText); @@ -753,7 +752,9 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw } serde.setParameters(new HashMap()); serde.setName((String)fields[12]); - serde.setSerializationLib((String)fields[13]); + // Check if the integer value is set: If yes, get the corresponding String, else use the String column. + String serdeLib = (int) fields[15] == 0 ? (String) fields[13] : SerdeLib.getSerdeFromNumber((int) fields[15]); + serde.setSerializationLib(serdeLib); serdeSb.append(serdeId).append(","); sd.setSerdeInfo(serde); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d27224b235..4f8e738bd8 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2032,8 +2032,12 @@ private SerDeInfo convertToSerDeInfo(MSerDeInfo ms) throws MetaException { if (ms == null) { throw new MetaException("Invalid SerDeInfo object"); } + + String serde_lib = ms.getSerializationLibInt() == 0 ? ms.getSerializationLib() : + SerdeLib.getSerdeFromNumber(ms.getSerializationLibInt()) ; + SerDeInfo serde = - new SerDeInfo(ms.getName(), ms.getSerializationLib(), convertMap(ms.getParameters())); + new SerDeInfo(ms.getName(), serde_lib, convertMap(ms.getParameters())); if (ms.getDescription() != null) { serde.setDescription(ms.getDescription()); } @@ -2053,9 +2057,23 @@ private MSerDeInfo convertToMSerDeInfo(SerDeInfo ms) throws MetaException { if (ms == null) { throw new MetaException("Invalid SerDeInfo object"); } - return new MSerDeInfo(ms.getName(), ms.getSerializationLib(), ms.getParameters(), + /* + * Check if there is an integer version for this serdeLib, if yes, String version is null and set + * serializationLibInt to the corresponding int + */ + + String serde_lib = SerdeLib.get(ms.getSerializationLib()) == 0 ? ms.getSerializationLib() : null; + + + MSerDeInfo serde = new MSerDeInfo(ms.getName(), serde_lib, ms.getParameters(), ms.getDescription(), ms.getSerializerClass(), ms.getDeserializerClass(), ms.getSerdeType() == null ? 0 : ms.getSerdeType().getValue()); + + if(serde_lib == null){ + serde.setSerializationLibInt(SerdeLib.get(ms.getSerializationLib())); + } + + return serde; } /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SerdeLib.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SerdeLib.java new file mode 100644 index 0000000000..4e3509b14f --- /dev/null +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SerdeLib.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * HIVE-20363: Have integer constants for frequently used serde classes. + */ + +public final class SerdeLib { + + private SerdeLib() { + } + + private static Map serdeMap = new HashMap() { + { + put("org.apache.hadoop.hive.serde2.avro.AvroSerDe", 1); + put("parquet.hive.serde.ParquetHiveSerDe", 2); + put("org.apache.hadoop.hive.serde2.OpenCSVSerde", 3); + put("org.apache.hadoop.hive.serde2.RegexSerDe", 4); + put("org.apache.hive.hcatalog.data.JsonSerDe", 5); + put("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", 6); + } + }; + + private static Map reverseSerdeMap = + serdeMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + /* + @return the String version of of the serdeNum + */ + public static String getSerdeFromNumber(int serdeNum) { + return reverseSerdeMap.getOrDefault(serdeNum, null); + } + + /* + @return the integer constant of the serdeLib + */ + public static int get(String serdeLib) { + return serdeMap.getOrDefault(serdeLib, 0); + } + +} diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java index 68f07e2569..d389c49b1c 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MSerDeInfo.java @@ -23,6 +23,7 @@ public class MSerDeInfo { private String name; private String serializationLib; + private int serializationLibInt; private Map parameters; private String description; private String serializerClass; @@ -51,6 +52,7 @@ public MSerDeInfo(String name, String serializationLib, Map para this.serdeType = serdeType; } + /** * @return the serializationLib */ @@ -124,4 +126,12 @@ public int getSerdeType() { public void setSerdeType(int serdeType) { this.serdeType = serdeType; } + + public int getSerializationLibInt() { + return serializationLibInt; + } + + public void setSerializationLibInt(int serializationLibInt) { + this.serializationLibInt = serializationLibInt; + } } diff --git standalone-metastore/metastore-server/src/main/resources/package.jdo standalone-metastore/metastore-server/src/main/resources/package.jdo index 2a5f016b1f..69f3be4ec3 100644 --- standalone-metastore/metastore-server/src/main/resources/package.jdo +++ standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -286,6 +286,9 @@ + + + diff --git standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 3af2ebb253..0fad92dd78 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -455,6 +455,7 @@ CREATE TABLE IF NOT EXISTS `SERDES` ( `SERDE_ID` bigint(20) NOT NULL, `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `SLIB` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SLIB_INT` int(2) DEFAULT NULL, `DESCRIPTION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `SERIALIZER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `DESERIALIZER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index ee0f691b52..1631f8d80b 100644 --- standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -4,7 +4,44 @@ SELECT 'Upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; ALTER TABLE TBLS ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE PARTITIONS ADD WRITE_ID bigint DEFAULT 0; +-- HIVE-20363 +ALTER TABLE SERDES ADD COLUMN SLIB_INT int(2) DEFAULT NULL; + +UPDATE SERDES +SET SLIB_INT = CASE WHEN SLIB='org.apache.hadoop.hive.serde2.avro.AvroSerDe' THEN 1 + WHEN SLIB='parquet.hive.serde.ParquetHiveSerDe' THEN 2 + WHEN SLIB='org.apache.hadoop.hive.serde2.OpenCSVSerde' THEN 3 + WHEN SLIB='org.apache.hadoop.hive.serde2.RegexSerDe' THEN 4 + WHEN SLIB='org.apache.hive.hcatalog.data.JsonSerDe' THEN 5 + WHEN SLIB='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' THEN 6 + ELSE 0 + END + ,SLIB = CASE WHEN SLIB='org.apache.hadoop.hive.serde2.avro.AvroSerDe' + OR SLIB='parquet.hive.serde.ParquetHiveSerDe' OR SLIB='org.apache.hadoop.hive.serde2.OpenCSVSerde' + OR SLIB='org.apache.hadoop.hive.serde2.RegexSerDe' OR SLIB='org.apache.hive.hcatalog.data.JsonSerDe' + OR SLIB='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' THEN NULL + ELSE SLIB + END; + +-- HIVE-20363 +ALTER TABLE SERDES ADD COLUMN SLIB_INT int(2) DEFAULT NULL; + +UPDATE SERDES +SET SLIB_INT = CASE WHEN SLIB='org.apache.hadoop.hive.serde2.avro.AvroSerDe' THEN 1 + WHEN SLIB='parquet.hive.serde.ParquetHiveSerDe' THEN 2 + WHEN SLIB='org.apache.hadoop.hive.serde2.OpenCSVSerde' THEN 3 + WHEN SLIB='org.apache.hadoop.hive.serde2.RegexSerDe' THEN 4 + WHEN SLIB='org.apache.hive.hcatalog.data.JsonSerDe' THEN 5 + WHEN SLIB='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' THEN 6 + ELSE 0 + END + ,SLIB = CASE WHEN SLIB='org.apache.hadoop.hive.serde2.avro.AvroSerDe' + OR SLIB='parquet.hive.serde.ParquetHiveSerDe' OR SLIB='org.apache.hadoop.hive.serde2.OpenCSVSerde' + OR SLIB='org.apache.hadoop.hive.serde2.RegexSerDe' OR SLIB='org.apache.hive.hcatalog.data.JsonSerDe' + OR SLIB='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' THEN NULL + ELSE SLIB + END; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; -SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; - +SELECT 'Finished upgrading MetaStore schema from 3.2.0 to 4.0.0' AS ' '; \ No newline at end of file