commit 2193448510b8dfe318b45012a2ab764a49d5248e Author: Daniel Dai Date: Wed Jun 27 01:10:22 2018 -0700 HIVE-19938: Upgrade scripts for information schema diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql index 9bfe9c2..f98894f 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql @@ -1,7 +1,6 @@ -- HIVE system db -DROP DATABASE IF EXISTS SYS CASCADE; -CREATE DATABASE SYS; +CREATE DATABASE IF NOT EXISTS SYS; USE SYS; @@ -817,14 +816,8 @@ FROM \"PART_COL_STATS\"" ); -CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( - `VER_ID` BIGINT, - `SCHEMA_VERSION` string, - `VERSION_COMMENT` string, - CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE -); - -INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0'); +CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`, + 'Hive release version 3.1.0' AS `VERSION_COMMENT`; CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, @@ -924,7 +917,7 @@ FROM \"KEY_CONSTRAINTS\"" ); -CREATE VIEW `TABLE_STATS_VIEW` AS +CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS SELECT `TBL_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -935,7 +928,7 @@ SELECT max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME FROM `TABLE_PARAMS` GROUP BY `TBL_ID`; -CREATE VIEW `PARTITION_STATS_VIEW` AS +CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS SELECT `PART_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -1064,12 +1057,11 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I " ); -DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE; -CREATE DATABASE INFORMATION_SCHEMA; +CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA; USE INFORMATION_SCHEMA; -CREATE VIEW IF NOT EXISTS `SCHEMATA` +CREATE OR REPLACE VIEW `SCHEMATA` ( `CATALOG_NAME`, `SCHEMA_NAME`, @@ -1097,7 +1089,7 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND current_authorizer() = P.`AUTHORIZER`; -CREATE VIEW IF NOT EXISTS `TABLES` +CREATE OR REPLACE VIEW `TABLES` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -1134,7 +1126,7 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP'))) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer(); -CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` +CREATE OR REPLACE VIEW `TABLE_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -1168,7 +1160,7 @@ WHERE OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP'))) AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER` = current_authorizer() AND P2.`AUTHORIZER` = current_authorizer(); -CREATE VIEW IF NOT EXISTS `COLUMNS` +CREATE OR REPLACE VIEW `COLUMNS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -1323,7 +1315,7 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_COL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()); -CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` +CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -1359,7 +1351,7 @@ WHERE OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) AND P2.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer() AND P2.`AUTHORIZER`=current_authorizer()); -CREATE VIEW IF NOT EXISTS `VIEWS` +CREATE OR REPLACE VIEW `VIEWS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, diff --git a/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql new file mode 100644 index 0000000..ac93a12 --- /dev/null +++ b/metastore/scripts/upgrade/hive/upgrade-3.0.0-to-3.1.0.hive.sql @@ -0,0 +1,10 @@ +SELECT 'Upgrading MetaStore schema from 3.0.0 to 3.1.0'; + +USE SYS; + +DROP TABLE IF EXISTS `VERSION`; + +CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '3.1.0' AS `SCHEMA_VERSION`, + 'Hive release version 3.0.0' AS `VERSION_COMMENT`; + +SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0'; diff --git a/metastore/scripts/upgrade/hive/upgrade.order.hive b/metastore/scripts/upgrade/hive/upgrade.order.hive index e69de29..dd040af 100644 --- a/metastore/scripts/upgrade/hive/upgrade.order.hive +++ b/metastore/scripts/upgrade/hive/upgrade.order.hive @@ -0,0 +1 @@ +3.0.0-to-3.1.0 diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index a9557cf..2dd9260 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -215,7 +215,8 @@ **/* - **/upgrade.order.* + + %regex[(!hive)/upgrade.order.*] scripts/metastore/upgrade diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 143b710..480b09a 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -66,14 +66,10 @@ default srcpart hive_test_user USER DELETE true -1 hive_test_user default srcpart hive_test_user USER INSERT true -1 hive_test_user default srcpart hive_test_user USER SELECT true -1 hive_test_user default srcpart hive_test_user USER UPDATE true -1 hive_test_user -PREHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE -PREHOOK: type: DROPDATABASE -POSTHOOK: query: DROP DATABASE IF EXISTS SYS CASCADE -POSTHOOK: type: DROPDATABASE -PREHOOK: query: CREATE DATABASE SYS +PREHOOK: query: CREATE DATABASE IF NOT EXISTS SYS PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:SYS -POSTHOOK: query: CREATE DATABASE SYS +POSTHOOK: query: CREATE DATABASE IF NOT EXISTS SYS POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:SYS PREHOOK: query: USE SYS @@ -1842,35 +1838,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PART_COL_STATS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( - `VER_ID` BIGINT, - `SCHEMA_VERSION` string, - `VERSION_COMMENT` string, - CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@VERSION -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( - `VER_ID` BIGINT, - `SCHEMA_VERSION` string, - `VERSION_COMMENT` string, - CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: SYS@VERSION -POSTHOOK: Output: database:sys -PREHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: sys@version -POSTHOOK: query: INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: sys@version -POSTHOOK: Lineage: version.schema_version SCRIPT [] -POSTHOOK: Lineage: version.ver_id SCRIPT [] -POSTHOOK: Lineage: version.version_comment SCRIPT [] +FAILED: SemanticException org.apache.hadoop.hive.ql.metadata.InvalidTableException: Table not found _dummy_table PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, @@ -2053,7 +2021,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@KEY_CONSTRAINTS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS +PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS SELECT `TBL_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -2067,7 +2035,7 @@ PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@table_params PREHOOK: Output: SYS@TABLE_STATS_VIEW PREHOOK: Output: database:sys -POSTHOOK: query: CREATE VIEW `TABLE_STATS_VIEW` AS +POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_STATS_VIEW` AS SELECT `TBL_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -2088,7 +2056,7 @@ POSTHOOK: Lineage: TABLE_STATS_VIEW.raw_data_size EXPRESSION [(table_params)tabl POSTHOOK: Lineage: TABLE_STATS_VIEW.tbl_id SIMPLE [(table_params)table_params.FieldSchema(name:tbl_id, type:bigint, comment:from deserializer), ] POSTHOOK: Lineage: TABLE_STATS_VIEW.total_size EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] POSTHOOK: Lineage: TABLE_STATS_VIEW.transient_last_ddl_time EXPRESSION [(table_params)table_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (table_params)table_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] -PREHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS +PREHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS SELECT `PART_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -2102,7 +2070,7 @@ PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@partition_params PREHOOK: Output: SYS@PARTITION_STATS_VIEW PREHOOK: Output: database:sys -POSTHOOK: query: CREATE VIEW `PARTITION_STATS_VIEW` AS +POSTHOOK: query: CREATE OR REPLACE VIEW `PARTITION_STATS_VIEW` AS SELECT `PART_ID`, max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, @@ -2379,14 +2347,10 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_MAPPINGS POSTHOOK: Output: database:sys -PREHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE -PREHOOK: type: DROPDATABASE -POSTHOOK: query: DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE -POSTHOOK: type: DROPDATABASE -PREHOOK: query: CREATE DATABASE INFORMATION_SCHEMA +PREHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:INFORMATION_SCHEMA -POSTHOOK: query: CREATE DATABASE INFORMATION_SCHEMA +POSTHOOK: query: CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:INFORMATION_SCHEMA PREHOOK: query: USE INFORMATION_SCHEMA @@ -2395,7 +2359,7 @@ PREHOOK: Input: database:information_schema POSTHOOK: query: USE INFORMATION_SCHEMA POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:information_schema -PREHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA` +PREHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA` ( `CATALOG_NAME`, `SCHEMA_NAME`, @@ -2428,7 +2392,7 @@ PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@SCHEMATA PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA` +POSTHOOK: query: CREATE OR REPLACE VIEW `SCHEMATA` ( `CATALOG_NAME`, `SCHEMA_NAME`, @@ -2468,7 +2432,7 @@ POSTHOOK: Lineage: SCHEMATA.default_character_set_schema EXPRESSION [] POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ] #### A masked pattern was here #### POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)d.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ] -PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES` +PREHOOK: query: CREATE OR REPLACE VIEW `TABLES` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -2510,7 +2474,7 @@ PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@TABLES PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES` +POSTHOOK: query: CREATE OR REPLACE VIEW `TABLES` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -2564,7 +2528,7 @@ POSTHOOK: Lineage: TABLES.table_type EXPRESSION [(tbls)t.FieldSchema(name:view_o POSTHOOK: Lineage: TABLES.user_defined_type_catalog EXPRESSION [] POSTHOOK: Lineage: TABLES.user_defined_type_name EXPRESSION [] POSTHOOK: Lineage: TABLES.user_defined_type_schema EXPRESSION [] -PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` +PREHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -2603,7 +2567,7 @@ PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@TABLE_PRIVILEGES PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` +POSTHOOK: query: CREATE OR REPLACE VIEW `TABLE_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -2650,7 +2614,7 @@ POSTHOOK: Lineage: TABLE_PRIVILEGES.table_catalog SIMPLE [] POSTHOOK: Lineage: TABLE_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ] POSTHOOK: Lineage: TABLE_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ] POSTHOOK: Lineage: TABLE_PRIVILEGES.with_hierarchy SIMPLE [] -PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS` +PREHOOK: query: CREATE OR REPLACE VIEW `COLUMNS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -2812,7 +2776,7 @@ PREHOOK: Input: sys@tbl_col_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@COLUMNS PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS` +POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMNS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -3021,7 +2985,7 @@ POSTHOOK: Lineage: COLUMNS.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, ty POSTHOOK: Lineage: COLUMNS.udt_catalog EXPRESSION [] POSTHOOK: Lineage: COLUMNS.udt_name EXPRESSION [] POSTHOOK: Lineage: COLUMNS.udt_schema EXPRESSION [] -PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` +PREHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -3064,7 +3028,7 @@ PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` +POSTHOOK: query: CREATE OR REPLACE VIEW `COLUMN_PRIVILEGES` ( `GRANTOR`, `GRANTEE`, @@ -3115,7 +3079,7 @@ POSTHOOK: Lineage: COLUMN_PRIVILEGES.privilege_type SIMPLE [(tbl_col_privs)p.Fie POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_catalog SIMPLE [] POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_name SIMPLE [(tbls)t.FieldSchema(name:tbl_name, type:string, comment:from deserializer), ] POSTHOOK: Lineage: COLUMN_PRIVILEGES.table_schema SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ] -PREHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS` +PREHOOK: query: CREATE OR REPLACE VIEW `VIEWS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, @@ -3157,7 +3121,7 @@ PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@VIEWS PREHOOK: Output: database:information_schema -POSTHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS` +POSTHOOK: query: CREATE OR REPLACE VIEW `VIEWS` ( `TABLE_CATALOG`, `TABLE_SCHEMA`, diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java index f868982..2cadab5 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java @@ -47,7 +47,8 @@ public static final String EMBEDDED_HS2_URL = "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;" + "hive.security.authorization.sqlstd.confwhitelist.append=.*;hive.security.authorization.enabled=false;" - + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory"; + + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory;" + + "hive.support.concurrency=false;hive.metastore.rawstore.impl=org.apache.hadoop.hive.metastore.ObjectStore"; public static final String HIVE_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver"; /***