diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 26c4937a28..dfc70a5999 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1378,4 +1378,9 @@ public int deleteScheduledExecutions(int maxRetainSecs) { public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException { return objectStore.markScheduledExecutionsTimedOut(timeoutSecs); } + + @Override + public List getTableMVUsage(String catName, String dbName, String tableName) { + return objectStore.getTableMVUsage(catName, dbName, tableName); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 1eb9c12cc8..60e1df4a4d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1235,13 +1235,6 @@ public void dropTable(String dbName, String tableName, boolean deleteData, if (!ignoreUnknownTab) { throw new HiveException(e); } - } catch (MetaException e) { - int idx = ExceptionUtils.indexOfType(e, SQLIntegrityConstraintViolationException.class); - if (idx != -1 && ExceptionUtils.getThrowables(e)[idx].getMessage().contains("MV_TABLES_USED")) { - throw new HiveException("Cannot drop table since it is used by at least one materialized view definition. " + - "Please drop any materialized view that uses the table before dropping it", e); - } - throw new HiveException(e); } catch (Exception e) { throw new HiveException(e); } diff --git ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q new file mode 100644 index 0000000000..04c3df6843 --- /dev/null +++ ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q @@ -0,0 +1,13 @@ +create table mytable (key int, value string); +insert into mytable values (1, 'val1'), (2, 'val2'); +create view myview as select * from mytable; + +create materialized view mv1 disable rewrite as +select key, value from myview; +create materialized view mv2 disable rewrite as +select count(*) from myview; + +-- dropping the view is fine, as the MV uses not the view itself, but it's query for creating it's own during it's creation +drop view myview; +drop table mytable; + diff --git ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out index 0a2020367e..e2dc96f697 100644 --- ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out +++ ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out @@ -32,4 +32,5 @@ PREHOOK: query: drop table mytable PREHOOK: type: DROPTABLE PREHOOK: Input: default@mytable PREHOOK: Output: default@mytable -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. Cannot drop table since it is used by at least one materialized view definition. Please drop any materialized view that uses the table before dropping it +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Cannot drop table since it is used by the following materialized view definition: [default.mv1] +Please drop them before dropping the table.) diff --git ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out new file mode 100644 index 0000000000..6b93ea527c --- /dev/null +++ ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: create table mytable (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mytable +POSTHOOK: query: create table mytable (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mytable +PREHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@mytable +POSTHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@mytable +POSTHOOK: Lineage: mytable.key SCRIPT [] +POSTHOOK: Lineage: mytable.value SCRIPT [] +PREHOOK: query: create view myview as select * from mytable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@mytable +PREHOOK: Output: database:default +PREHOOK: Output: default@myview +POSTHOOK: query: create view myview as select * from mytable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myview +POSTHOOK: Lineage: myview.key SIMPLE [(mytable)mytable.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: myview.value SIMPLE [(mytable)mytable.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv2 +POSTHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv2 +PREHOOK: query: drop view myview +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@myview +PREHOOK: Output: default@myview +POSTHOOK: query: drop view myview +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@myview +POSTHOOK: Output: default@myview +PREHOOK: query: drop table mytable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@mytable +PREHOOK: Output: default@mytable +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Cannot drop table since it is used by the following materialized view definitions: [default.mv1, default.mv2] +Please drop them before dropping the table.) diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index c88c889c50..6ab03d2644 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2738,7 +2738,7 @@ private boolean is_table_exists(RawStore ms, String catName, String dbname, Stri return (ms.getTable(catName, dbname, name, null) != null); } - private boolean drop_table_core(final RawStore ms, final String catName, final String dbname, + private boolean drop_table_core(final RawStore ms, final String catName, final String dbName, final String name, final boolean deleteData, final EnvironmentContext envContext, final String indexName) throws NoSuchObjectException, MetaException, IOException, InvalidObjectException, @@ -2754,11 +2754,11 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S boolean isReplicated = false; try { ms.openTransaction(); - db = ms.getDatabase(catName, dbname); + db = ms.getDatabase(catName, dbName); isReplicated = isDbReplicationTarget(db); // drop any partitions - tbl = get_table_core(catName, dbname, name); + tbl = get_table_core(catName, dbName, name); if (tbl == null) { throw new NoSuchObjectException(name + " doesn't exist"); } @@ -2767,6 +2767,14 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S } ifPurge = isMustPurge(envContext, tbl); + List mvs = ms.getTableMVUsage(catName, dbName, name); + if (!mvs.isEmpty()) { + String plural = mvs.size() == 1 ? "" : "s"; + throw new MetaException(String.format( + "Cannot drop table since it is used by the following materialized view definition%s: %s%n" + + "Please drop them before dropping the table.", plural, mvs)); + } + firePreEvent(new PreDropTableEvent(tbl, deleteData, this)); tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(tbl, deleteData); @@ -2781,14 +2789,14 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S } // Drop the partitions and get a list of locations which need to be deleted - partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath, + partPaths = dropPartitionsAndGetLocations(ms, catName, dbName, name, tblPath, tableDataShouldBeDeleted); // Drop any constraints on the table - ms.dropConstraint(catName, dbname, name, null, true); + ms.dropConstraint(catName, dbName, name, null, true); - if (!ms.dropTable(catName, dbname, name)) { - String tableName = TableName.getQualified(catName, dbname, name); + if (!ms.dropTable(catName, dbName, name)) { + String tableName = TableName.getQualified(catName, dbName, name); throw new MetaException(indexName == null ? "Unable to drop table " + tableName: "Unable to drop index table " + tableName + " for index " + indexName); } else { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 1a5944d33e..2adc24d091 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1681,6 +1681,30 @@ private MCreationMetadata getCreationMetadata(String catName, String dbName, Str return mcm; } + @Override + public List getTableMVUsage(String catName, String dbName, String tableName) { + boolean commited = false; + List mvs = new ArrayList<>(); + Query query = null; + try { + openTransaction(); + MTable table = getMTable(catName, dbName, tableName); + query = pm.newQuery( + MCreationMetadata.class, "tables.contains(table)"); + query.declareParameters("org.apache.hadoop.hive.metastore.model.MTable table"); + query.setParameters(table); + List mcms = query.executeList(); + for (MCreationMetadata mcm : mcms) { + pm.retrieve(mcm); + mvs.add(mcm.getDbName() + "." + mcm.getTblName()); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return mvs; + } + private MTable getMTable(String catName, String db, String table) { AttachedMTableInfo nmtbl = getMTable(catName, db, table, false); return nmtbl.mtbl; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 09850c50ef..08754d77e8 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.model.MCreationMetadata; +import org.apache.hadoop.hive.metastore.model.MTable; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -1821,4 +1823,6 @@ void scheduledQueryProgress(ScheduledQueryProgressInfo info) int deleteScheduledExecutions(int maxRetainSecs); int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException; + + List getTableMVUsage(String catName, String dbName, String tableName); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 4d8dc4c047..c56308fd0d 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; @@ -2893,4 +2894,9 @@ public int deleteScheduledExecutions(int maxRetainSecs) { public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException { return rawStore.markScheduledExecutionsTimedOut(timeoutSecs); } + + @Override + public List getTableMVUsage(String catName, String dbName, String tableName) { + return rawStore.getTableMVUsage(catName, dbName, tableName); + } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 6a6ba5fb13..9a7d705257 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; +import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; @@ -1327,4 +1328,9 @@ public int deleteScheduledExecutions(int maxRetainSecs) { public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException { return objectStore.markScheduledExecutionsTimedOut(timeoutSecs); } + + @Override + public List getTableMVUsage(String catName, String dbName, String tableName) { + return objectStore.getTableMVUsage(catName, dbName, tableName); + } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 1f0bb47929..b4d9d646ca 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -93,6 +93,7 @@ import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.thrift.TException; @@ -1311,4 +1312,9 @@ public int deleteScheduledExecutions(int maxRetainSecs) { public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException{ throw new RuntimeException("unimplemented"); } + + @Override + public List getTableMVUsage(String catName, String dbName, String tableName) { + throw new RuntimeException("unimplemented"); + } }