diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index dcd705668c..f390dc5ddd 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -497,6 +497,7 @@ minillaplocal.query.files=\ compare_double_bigint_2.q,\ constprog_dpp.q,\ constant_prop_when.q,\ + constraints_alter.q,\ constraints_optimization.q,\ current_date_timestamp.q,\ correlationoptimizer1.q,\ diff --git a/ql/src/test/queries/clientpositive/constraints_alter.q b/ql/src/test/queries/clientpositive/constraints_alter.q new file mode 100644 index 0000000000..ab2c4d37a5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/constraints_alter.q @@ -0,0 +1,88 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint, + `COMMENT` string, + `COLUMN_NAME` string, + `TYPE_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_COLUMN_V2` PRIMARY KEY (`CD_ID`,`COLUMN_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CD_ID\", + \"COMMENT\", + \"COLUMN_NAME\", + \"TYPE_NAME\", + \"INTEGER_IDX\" +FROM + \"COLUMNS_V2\"" +); + +CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +( + `CHILD_CD_ID` bigint, + `CHILD_INTEGER_IDX` int, + `CHILD_TBL_ID` bigint, + `PARENT_CD_ID` bigint, + `PARENT_INTEGER_IDX` int, + `PARENT_TBL_ID` bigint, + `POSITION` bigint, + `CONSTRAINT_NAME` string, + `CONSTRAINT_TYPE` string, + `UPDATE_RULE` string, + `DELETE_RULE` string, + `ENABLE_VALIDATE_RELY` int, + `DEFAULT_VALUE` string, + CONSTRAINT `SYS_PK_KEY_CONSTRAINTS` PRIMARY KEY (`CONSTRAINT_NAME`, `POSITION`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CHILD_CD_ID\", + \"CHILD_INTEGER_IDX\", + \"CHILD_TBL_ID\", + \"PARENT_CD_ID\", + \"PARENT_INTEGER_IDX\", + \"PARENT_TBL_ID\", + \"POSITION\", + \"CONSTRAINT_NAME\", + \"CONSTRAINT_TYPE\", + \"UPDATE_RULE\", + \"DELETE_RULE\", + \"ENABLE_VALIDATE_RELY\", + \"DEFAULT_VALUE\" +FROM + \"KEY_CONSTRAINTS\"" +); + + +create table t1_n2939 (a_n2939 integer, b_n2939 string); + +insert into table t1_n2939 values (1, '1'), (2, '2'); + +select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939'; +select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2'; + +alter table t1_n2939 add constraint t1_n2939_pk primary key (a_n2939) disable novalidate rely; + +select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939'; +select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2'; + +alter table t1_n2939 change column b_n2939 b_n2939 string constraint t1_n2939_nn not null disable novalidate rely; + +select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939'; +select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2'; + +alter table t1_n2939 change column b_n2939 c_n2939 string constraint t1_n2939_nn_2 not null disable novalidate rely; + +select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939'; +select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2'; + +drop table t1_n2939; + +select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939'; +select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2'; diff --git a/ql/src/test/results/clientpositive/llap/constraints_alter.q.out b/ql/src/test/results/clientpositive/llap/constraints_alter.q.out new file mode 100644 index 0000000000..a66d148a14 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/constraints_alter.q.out @@ -0,0 +1,269 @@ +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint, + `COMMENT` string, + `COLUMN_NAME` string, + `TYPE_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_COLUMN_V2` PRIMARY KEY (`CD_ID`,`COLUMN_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CD_ID\", + \"COMMENT\", + \"COLUMN_NAME\", + \"TYPE_NAME\", + \"INTEGER_IDX\" +FROM + \"COLUMNS_V2\"" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@COLUMNS_V2 +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint, + `COMMENT` string, + `COLUMN_NAME` string, + `TYPE_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_COLUMN_V2` PRIMARY KEY (`CD_ID`,`COLUMN_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CD_ID\", + \"COMMENT\", + \"COLUMN_NAME\", + \"TYPE_NAME\", + \"INTEGER_IDX\" +FROM + \"COLUMNS_V2\"" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@COLUMNS_V2 +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +( + `CHILD_CD_ID` bigint, + `CHILD_INTEGER_IDX` int, + `CHILD_TBL_ID` bigint, + `PARENT_CD_ID` bigint, + `PARENT_INTEGER_IDX` int, + `PARENT_TBL_ID` bigint, + `POSITION` bigint, + `CONSTRAINT_NAME` string, + `CONSTRAINT_TYPE` string, + `UPDATE_RULE` string, + `DELETE_RULE` string, + `ENABLE_VALIDATE_RELY` int, + `DEFAULT_VALUE` string, + CONSTRAINT `SYS_PK_KEY_CONSTRAINTS` PRIMARY KEY (`CONSTRAINT_NAME`, `POSITION`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CHILD_CD_ID\", + \"CHILD_INTEGER_IDX\", + \"CHILD_TBL_ID\", + \"PARENT_CD_ID\", + \"PARENT_INTEGER_IDX\", + \"PARENT_TBL_ID\", + \"POSITION\", + \"CONSTRAINT_NAME\", + \"CONSTRAINT_TYPE\", + \"UPDATE_RULE\", + \"DELETE_RULE\", + \"ENABLE_VALIDATE_RELY\", + \"DEFAULT_VALUE\" +FROM + \"KEY_CONSTRAINTS\"" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@KEY_CONSTRAINTS +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +( + `CHILD_CD_ID` bigint, + `CHILD_INTEGER_IDX` int, + `CHILD_TBL_ID` bigint, + `PARENT_CD_ID` bigint, + `PARENT_INTEGER_IDX` int, + `PARENT_TBL_ID` bigint, + `POSITION` bigint, + `CONSTRAINT_NAME` string, + `CONSTRAINT_TYPE` string, + `UPDATE_RULE` string, + `DELETE_RULE` string, + `ENABLE_VALIDATE_RELY` int, + `DEFAULT_VALUE` string, + CONSTRAINT `SYS_PK_KEY_CONSTRAINTS` PRIMARY KEY (`CONSTRAINT_NAME`, `POSITION`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CHILD_CD_ID\", + \"CHILD_INTEGER_IDX\", + \"CHILD_TBL_ID\", + \"PARENT_CD_ID\", + \"PARENT_INTEGER_IDX\", + \"PARENT_TBL_ID\", + \"POSITION\", + \"CONSTRAINT_NAME\", + \"CONSTRAINT_TYPE\", + \"UPDATE_RULE\", + \"DELETE_RULE\", + \"ENABLE_VALIDATE_RELY\", + \"DEFAULT_VALUE\" +FROM + \"KEY_CONSTRAINTS\"" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@KEY_CONSTRAINTS +PREHOOK: query: create table t1_n2939 (a_n2939 integer, b_n2939 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n2939 +POSTHOOK: query: create table t1_n2939 (a_n2939 integer, b_n2939 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n2939 +PREHOOK: query: insert into table t1_n2939 values (1, '1'), (2, '2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1_n2939 +POSTHOOK: query: insert into table t1_n2939 values (1, '1'), (2, '2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1_n2939 +POSTHOOK: Lineage: t1_n2939.a_n2939 SCRIPT [] +POSTHOOK: Lineage: t1_n2939.b_n2939 SCRIPT [] +PREHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +PREHOOK: type: QUERY +PREHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +POSTHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +a_n2939 +b_n2939 +PREHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +PREHOOK: type: QUERY +PREHOOK: Input: default@key_constraints +#### A masked pattern was here #### +POSTHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@key_constraints +#### A masked pattern was here #### +PREHOOK: query: alter table t1_n2939 add constraint t1_n2939_pk primary key (a_n2939) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table t1_n2939 add constraint t1_n2939_pk primary key (a_n2939) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +PREHOOK: type: QUERY +PREHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +POSTHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +a_n2939 +b_n2939 +PREHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +PREHOOK: type: QUERY +PREHOOK: Input: default@key_constraints +#### A masked pattern was here #### +POSTHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@key_constraints +#### A masked pattern was here #### +t1_n2939_pk +PREHOOK: query: alter table t1_n2939 change column b_n2939 b_n2939 string constraint t1_n2939_nn not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@t1_n2939 +PREHOOK: Output: default@t1_n2939 +POSTHOOK: query: alter table t1_n2939 change column b_n2939 b_n2939 string constraint t1_n2939_nn not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@t1_n2939 +POSTHOOK: Output: default@t1_n2939 +PREHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +PREHOOK: type: QUERY +PREHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +POSTHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +a_n2939 +b_n2939 +PREHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +PREHOOK: type: QUERY +PREHOOK: Input: default@key_constraints +#### A masked pattern was here #### +POSTHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@key_constraints +#### A masked pattern was here #### +t1_n2939_pk +t1_n2939_nn +PREHOOK: query: alter table t1_n2939 change column b_n2939 c_n2939 string constraint t1_n2939_nn_2 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@t1_n2939 +PREHOOK: Output: default@t1_n2939 +POSTHOOK: query: alter table t1_n2939 change column b_n2939 c_n2939 string constraint t1_n2939_nn_2 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@t1_n2939 +POSTHOOK: Output: default@t1_n2939 +PREHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +PREHOOK: type: QUERY +PREHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +POSTHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +a_n2939 +c_n2939 +PREHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +PREHOOK: type: QUERY +PREHOOK: Input: default@key_constraints +#### A masked pattern was here #### +POSTHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@key_constraints +#### A masked pattern was here #### +t1_n2939_pk +t1_n2939_nn_2 +PREHOOK: query: drop table t1_n2939 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1_n2939 +PREHOOK: Output: default@t1_n2939 +POSTHOOK: query: drop table t1_n2939 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1_n2939 +POSTHOOK: Output: default@t1_n2939 +PREHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +PREHOOK: type: QUERY +PREHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +POSTHOOK: query: select COLUMN_NAME from columns_v2 where column_name = 'a_n2939' or column_name = 'b_n2939' or column_name = 'c_n2939' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columns_v2 +#### A masked pattern was here #### +PREHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +PREHOOK: type: QUERY +PREHOOK: Input: default@key_constraints +#### A masked pattern was here #### +POSTHOOK: query: select CONSTRAINT_NAME from key_constraints where constraint_name = 't1_n2939_pk' or constraint_name = 't1_n2939_nn' or constraint_name = 't1_n2939_nn_2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@key_constraints +#### A masked pattern was here #### diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index def32c7edb..12bdb6420e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4474,14 +4474,52 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { // Convert the MFieldSchema's to their thrift object counterparts, because we maintain // datastore identity (i.e., identity of the model objects are managed by JDO, // not the application). - if (!(oldSd != null && oldSd.getCD() != null && - oldSd.getCD().getCols() != null && - newSd != null && newSd.getCD() != null && - newSd.getCD().getCols() != null && - convertToFieldSchemas(newSd.getCD().getCols()). - equals(convertToFieldSchemas(oldSd.getCD().getCols())) - )) { - oldSd.setCD(newSd.getCD()); + List oldCols = oldSd != null && oldSd.getCD() != null && oldSd.getCD().getCols() != null ? + convertToFieldSchemas(oldSd.getCD().getCols()) : null; + List newCols = newSd != null && newSd.getCD() != null && newSd.getCD().getCols() != null ? + convertToFieldSchemas(newSd.getCD().getCols()) : null; + if (oldCols == null || !oldCols.equals(newCols)) { + // First replace any constraints that may be associated with this CD + // Create mapping from old col indexes to new col indexes + if (oldCols != null && newCols != null) { + Map mapping = new HashMap<>(); + for (int i = 0; i < oldCols.size(); i++) { + FieldSchema oldCol = oldCols.get(i); + for (int j = 0; j < newCols.size(); j++) { + FieldSchema newCol = newCols.get(j); + if (oldCol.equals(newCol)) { + mapping.put(i, j); + break; + } + } + } + // If we find it, we will change the reference for the CD. + // If we do not find it, i.e., the column will be deleted, we do not change it + // and we let the logic in removeUnusedColumnDescriptor take care of it + Query query = pm.newQuery(MConstraint.class, "parentColumn == inCD || childColumn == inCD"); + query.declareParameters("MColumnDescriptor inCD"); + List mConstraintsList = (List) query.execute(oldSd.getCD()); + pm.retrieveAll(mConstraintsList); + for (MConstraint mConstraint : mConstraintsList) { + if (oldSd.getCD().equals(mConstraint.getParentColumn())) { + Integer newIdx = mapping.get(mConstraint.getParentIntegerIndex()); + if (newIdx != null) { + mConstraint.setParentColumn(newSd.getCD()); + mConstraint.setParentIntegerIndex(newIdx); + } + } + if (oldSd.getCD().equals(mConstraint.getChildColumn())) { + Integer newIdx = mapping.get(mConstraint.getChildIntegerIndex()); + if (newIdx != null) { + mConstraint.setChildColumn(newSd.getCD()); + mConstraint.setChildIntegerIndex(newIdx); + } + } + } + pm.makePersistentAll(mConstraintsList); + } + // Finally replace CD + oldSd.setCD(newSd.getCD()); } oldSd.setBucketCols(newSd.getBucketCols()); @@ -4525,6 +4563,14 @@ private void removeUnusedColumnDescriptor(MColumnDescriptor oldCD) { //if no other SD references this CD, we can throw it out. if (count == 0) { + // First remove any constraints that may be associated with this CD + query = pm.newQuery(MConstraint.class, "parentColumn == inCD || childColumn == inCD"); + query.declareParameters("MColumnDescriptor inCD"); + List mConstraintsList = (List) query.execute(oldCD); + if (CollectionUtils.isNotEmpty(mConstraintsList)) { + pm.deletePersistentAll(mConstraintsList); + } + // Finally remove CD pm.retrieve(oldCD); pm.deletePersistent(oldCD); }