diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index c872e69921..e5d3d635ee 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1869,7 +1869,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," + "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," + "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + - "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe", + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + + "org.apache.hadoop.hive.serde2.OpenCSVSerde", "SerDes retrieving schema from metastore. This is an internal parameter."), @Deprecated diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 94467a41a4..9bd0dcdb2f 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -457,6 +457,7 @@ minillaplocal.query.files=\ acid_vectorization_original.q,\ alter_merge_stats_orc.q,\ alter_partition_change_col.q,\ + alter6.q,\ authorization_view_8.q,\ auto_join30.q,\ auto_join_filters.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index b9bb3abbe5..d20dc8b594 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; @@ -1064,7 +1065,7 @@ public void setTemporary(boolean isTemporary) { public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) { return StringUtils.isEmpty(serdeLib) || - conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib); + conf.getStringCollection(MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA.getVarname()).contains(serdeLib); } public static boolean shouldStoreFieldsInMetastore( diff --git a/ql/src/test/queries/clientpositive/alter6.q b/ql/src/test/queries/clientpositive/alter6.q new file mode 100644 index 0000000000..f0763562f5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/alter6.q @@ -0,0 +1,4 @@ +CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from deserializer', source_system_instrument_type string COMMENT 'from deserializer', source_system_instrument_sub_type string COMMENT 'from deserializer', settlement_currency string COMMENT 'from deserializer', quantity string COMMENT 'from deserializer', parvalue string COMMENT 'from deserializer', ifrs9_instrument_classification string COMMENT 'from deserializer', outstanding_amount string COMMENT 'from deserializer', principal_repayment_factor string COMMENT 'from deserializer', day_count_fraction string COMMENT 'from deserializer', cusip string COMMENT 'from deserializer', figi string COMMENT 'from deserializer', issuer_name string COMMENT 'from deserializer', notional_amount_currency string COMMENT 'from deserializer', settlement_date_of_near_transaction string COMMENT 'from deserializer', custodian_account_id string COMMENT 'from deserializer') PARTITIONED BY (reporttype string, source string, eodlevel string, currentbpd string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; +alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE; +select * from t3_br_eod_custody; +show create table t3_br_eod_custody; diff --git a/ql/src/test/results/clientpositive/llap/alter6.q.out b/ql/src/test/results/clientpositive/llap/alter6.q.out new file mode 100644 index 0000000000..4d6c59efcd --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/alter6.q.out @@ -0,0 +1,71 @@ +PREHOOK: query: CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from deserializer', source_system_instrument_type string COMMENT 'from deserializer', source_system_instrument_sub_type string COMMENT 'from deserializer', settlement_currency string COMMENT 'from deserializer', quantity string COMMENT 'from deserializer', parvalue string COMMENT 'from deserializer', ifrs9_instrument_classification string COMMENT 'from deserializer', outstanding_amount string COMMENT 'from deserializer', principal_repayment_factor string COMMENT 'from deserializer', day_count_fraction string COMMENT 'from deserializer', cusip string COMMENT 'from deserializer', figi string COMMENT 'from deserializer', issuer_name string COMMENT 'from deserializer', notional_amount_currency string COMMENT 'from deserializer', settlement_date_of_near_transaction string COMMENT 'from deserializer', custodian_account_id string COMMENT 'from deserializer') PARTITIONED BY (reporttype string, source string, eodlevel string, currentbpd string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3_br_eod_custody +POSTHOOK: query: CREATE TABLE t3_br_eod_custody(data_publication_date_time string COMMENT 'from deserializer', booking_entity_name string COMMENT 'from deserializer', booking_entity_country_iso_code string COMMENT 'from deserializer', booking_entity_trade_portfolio_name string COMMENT 'from deserializer', isin string COMMENT 'from deserializer', coupon_rate string COMMENT 'from deserializer', instrument_description string COMMENT 'from deserializer', maturity_date string COMMENT 'from deserializer', source_system_instrument_type string COMMENT 'from deserializer', source_system_instrument_sub_type string COMMENT 'from deserializer', settlement_currency string COMMENT 'from deserializer', quantity string COMMENT 'from deserializer', parvalue string COMMENT 'from deserializer', ifrs9_instrument_classification string COMMENT 'from deserializer', outstanding_amount string COMMENT 'from deserializer', principal_repayment_factor string COMMENT 'from deserializer', day_count_fraction string COMMENT 'from deserializer', cusip string COMMENT 'from deserializer', figi string COMMENT 'from deserializer', issuer_name string COMMENT 'from deserializer', notional_amount_currency string COMMENT 'from deserializer', settlement_date_of_near_transaction string COMMENT 'from deserializer', custodian_account_id string COMMENT 'from deserializer') PARTITIONED BY (reporttype string, source string, eodlevel string, currentbpd string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3_br_eod_custody +PREHOOK: query: alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@t3_br_eod_custody +PREHOOK: Output: default@t3_br_eod_custody +POSTHOOK: query: alter table t3_br_eod_custody add columns (HKMA_Instrument_Id string) CASCADE +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@t3_br_eod_custody +POSTHOOK: Output: default@t3_br_eod_custody +PREHOOK: query: select * from t3_br_eod_custody +PREHOOK: type: QUERY +PREHOOK: Input: default@t3_br_eod_custody +#### A masked pattern was here #### +POSTHOOK: query: select * from t3_br_eod_custody +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t3_br_eod_custody +#### A masked pattern was here #### +PREHOOK: query: show create table t3_br_eod_custody +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@t3_br_eod_custody +POSTHOOK: query: show create table t3_br_eod_custody +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@t3_br_eod_custody +CREATE TABLE `t3_br_eod_custody`( + `data_publication_date_time` string COMMENT 'from deserializer', + `booking_entity_name` string COMMENT 'from deserializer', + `booking_entity_country_iso_code` string COMMENT 'from deserializer', + `booking_entity_trade_portfolio_name` string COMMENT 'from deserializer', + `isin` string COMMENT 'from deserializer', + `coupon_rate` string COMMENT 'from deserializer', + `instrument_description` string COMMENT 'from deserializer', + `maturity_date` string COMMENT 'from deserializer', + `source_system_instrument_type` string COMMENT 'from deserializer', + `source_system_instrument_sub_type` string COMMENT 'from deserializer', + `settlement_currency` string COMMENT 'from deserializer', + `quantity` string COMMENT 'from deserializer', + `parvalue` string COMMENT 'from deserializer', + `ifrs9_instrument_classification` string COMMENT 'from deserializer', + `outstanding_amount` string COMMENT 'from deserializer', + `principal_repayment_factor` string COMMENT 'from deserializer', + `day_count_fraction` string COMMENT 'from deserializer', + `cusip` string COMMENT 'from deserializer', + `figi` string COMMENT 'from deserializer', + `issuer_name` string COMMENT 'from deserializer', + `notional_amount_currency` string COMMENT 'from deserializer', + `settlement_date_of_near_transaction` string COMMENT 'from deserializer', + `custodian_account_id` string COMMENT 'from deserializer', + `hkma_instrument_id` string) +PARTITIONED BY ( + `reporttype` string, + `source` string, + `eodlevel` string, + `currentbpd` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.OpenCSVSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 58b67e888c..fc6a2fd43a 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -962,7 +962,8 @@ public static ConfVars getMetaConf(String name) { "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," + "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," + "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + - "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe", + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + + "org.apache.hadoop.hive.serde2.OpenCSVSerde", "SerDes retrieving schema from metastore. This is an internal parameter."), SERVER_MAX_MESSAGE_SIZE("metastore.server.max.message.size", "hive.metastore.server.max.message.size", 100*1024*1024L,