diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 26c4937a28..4984138ef8 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -269,6 +269,11 @@ public boolean dropTable(String catName, String dbName, String tableName) } } + @Override + public List isPartOfMaterializedView(String catName, String dbName, String tblName) { + return objectStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @Override public Table getTable(String catName, String dbName, String tableName) throws MetaException { return objectStore.getTable(catName, dbName, tableName); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 3b0b56d0bd..1f9fb3b897 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1235,13 +1235,6 @@ public void dropTable(String dbName, String tableName, boolean deleteData, if (!ignoreUnknownTab) { throw new HiveException(e); } - } catch (MetaException e) { - int idx = ExceptionUtils.indexOfType(e, SQLIntegrityConstraintViolationException.class); - if (idx != -1 && ExceptionUtils.getThrowables(e)[idx].getMessage().contains("MV_TABLES_USED")) { - throw new HiveException("Cannot drop table since it is used by at least one materialized view definition. " + - "Please drop any materialized view that uses the table before dropping it", e); - } - throw new HiveException(e); } catch (Exception e) { throw new HiveException(e); } diff --git ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q new file mode 100644 index 0000000000..458cc9ea94 --- /dev/null +++ ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q @@ -0,0 +1,12 @@ +create table mytable (key int, value string); +insert into mytable values (1, 'val1'), (2, 'val2'); +create view myview as select * from mytable; + +create materialized view mv1 disable rewrite as +select key, value from myview; +create materialized view mv2 disable rewrite as +select count(*) from myview; + +-- dropping the view is fine, as the MV uses not the view itself, but it's query for creating it's own during it's creation +drop view myview; +drop table mytable; diff --git ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out index 0a2020367e..5d980c13b1 100644 --- ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out +++ ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out @@ -32,4 +32,5 @@ PREHOOK: query: drop table mytable PREHOOK: type: DROPTABLE PREHOOK: Input: default@mytable PREHOOK: Output: default@mytable -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. Cannot drop table since it is used by at least one materialized view definition. Please drop any materialized view that uses the table before dropping it +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1] +) diff --git ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out new file mode 100644 index 0000000000..53ae0f86a4 --- /dev/null +++ ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: create table mytable (key int, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mytable +POSTHOOK: query: create table mytable (key int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mytable +PREHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@mytable +POSTHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@mytable +POSTHOOK: Lineage: mytable.key SCRIPT [] +POSTHOOK: Lineage: mytable.value SCRIPT [] +PREHOOK: query: create view myview as select * from mytable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@mytable +PREHOOK: Output: database:default +PREHOOK: Output: default@myview +POSTHOOK: query: create view myview as select * from mytable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@myview +POSTHOOK: Lineage: myview.key SIMPLE [(mytable)mytable.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: myview.value SIMPLE [(mytable)mytable.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 disable rewrite as +select key, value from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mytable +PREHOOK: Input: default@myview +PREHOOK: Output: database:default +PREHOOK: Output: default@mv2 +POSTHOOK: query: create materialized view mv2 disable rewrite as +select count(*) from myview +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mytable +POSTHOOK: Input: default@myview +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv2 +PREHOOK: query: drop view myview +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@myview +PREHOOK: Output: default@myview +POSTHOOK: query: drop view myview +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@myview +POSTHOOK: Output: default@myview +PREHOOK: query: drop table mytable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@mytable +PREHOOK: Output: default@mytable +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1, default.mv2] +) diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index be4b3794de..0ca614a422 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2762,6 +2762,15 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S if (tbl == null) { throw new NoSuchObjectException(name + " doesn't exist"); } + + // Check if table is part of a materialized view. + // If it is, it cannot be dropped. + List isPartOfMV = ms.isPartOfMaterializedView(catName, dbname, name); + if (!isPartOfMV.isEmpty()) { + throw new MetaException(String.format("Cannot drop table as it is used in the following materialized" + + " views %s%n", isPartOfMV)); + } + if (tbl.getSd() == null) { throw new MetaException("Table metadata is corrupted"); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ffc86071b8..289a3ee895 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1160,6 +1160,44 @@ private boolean dropCreationMetadata(String catName, String dbName, String table return success; } + @Override + public List isPartOfMaterializedView(String catName, String dbName, String tblName) { + + boolean committed = false; + Query query = null; + List mViewList = new ArrayList<>(); + + try { + openTransaction(); + + query = pm.newQuery("select from org.apache.hadoop.hive.metastore.model.MCreationMetadata"); + + List creationMetadata = (List)query.execute(); + Iterator iter = creationMetadata.iterator(); + + while (iter.hasNext()) + { + MCreationMetadata p = iter.next(); + Set tables = p.getTables(); + for (MTable table : tables) { + if (dbName.equals(table.getDatabase().getName()) && tblName.equals(table.getTableName())) { + LOG.info("Cannot drop table " + table.getTableName() + + " as it is being used by MView " + p.getTblName()); + mViewList.add(p.getDbName() + "." + p.getTblName()); + } + } + } + + committed = commitTransaction(); + } finally { + rollbackAndCleanup(committed, query); + } + return mViewList; + } + + + + private List listAllTableConstraintsWithOptionalConstraintName( String catName, String dbName, String tableName, String constraintname) { catName = normalizeIdentifier(catName); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 09850c50ef..c334421adf 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1792,6 +1792,15 @@ void alterSchemaVersion(SchemaVersionDescriptor version, SchemaVersion newVersio */ List getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException; + /** + * Checking if table is part of a materialized view. + * @param catName catalog the table is in + * @param dbName database the table is in + * @param tblName table name + * @return list of materialized views that uses the table + */ + List isPartOfMaterializedView(String catName, String dbName, String tblName); + /** * Returns details about a scheduled query by name. * diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 4d8dc4c047..f31cc5d7a1 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2862,6 +2862,10 @@ static boolean isBlacklistWhitelistEmpty(Configuration conf) { return rawStore.getPartitionColsWithStats(catName, dbName, tableName); } + @Override public List isPartOfMaterializedView(String catName, String dbName, String tblName) { + return rawStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @Override public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) throws MetaException { return rawStore.scheduledQueryPoll(request); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 6a6ba5fb13..f7032b93d1 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1290,6 +1290,12 @@ public void cleanWriteNotificationEvents(int olderThan) { return null; } + @Override + public List isPartOfMaterializedView(String catName, String dbName, String tblName) { + + return objectStore.isPartOfMaterializedView(catName, dbName, tblName); + } + @Override public Map> getPartitionColsWithStats(String catName, String dbName, String tableName) throws MetaException, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 1f0bb47929..bea7e9572b 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1284,6 +1284,11 @@ public void cleanWriteNotificationEvents(int olderThan) { return null; } + @Override + public List isPartOfMaterializedView(String catName, String dbName, String tblName) { + throw new RuntimeException("unimplemented"); + } + @Override public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) { throw new RuntimeException("unimplemented");