diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 511f5d6..373a1ec 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -785,6 +785,7 @@ rfc5424_parser_file_pruning.q,\ sysdb.q,\ sysdb_schq.q,\ + schq_materialized.q,\ table_access_keys_stats.q,\ temp_table_llap_partitioned.q,\ tez_bmj_schema_evolution.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 342d463..5191800 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lock.CompileLock; import org.apache.hadoop.hive.ql.lock.CompileLockFactory; +import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; @@ -352,7 +353,7 @@ // or a shared write lock if ((lckCmp.getType() == LockType.EXCLUSIVE || lckCmp.getType() == LockType.SHARED_WRITE) && - lckCmp.getTablename() != null) { + lckCmp.getTablename() != null && lckCmp.getDbname() != DbTxnManager.GLOBAL_LOCKS) { nonSharedLocks.add( TableName.getDbTable( lckCmp.getDbname(), lckCmp.getTablename())); diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java index a9efc1d..4641cd8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rebuild; +import org.antlr.runtime.tree.Tree; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.ql.Context; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +56,12 @@ return; } - TableName tableName = getQualifiedTableName((ASTNode) root.getChild(0)); + ASTNode tableTree = (ASTNode) root.getChild(0); + TableName tableName = getQualifiedTableName(tableTree); + if (ctx.enableUnparse()) { + unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase()); + return; + } ASTNode rewrittenAST = getRewrittenAST(tableName); mvRebuildMode = MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD; diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/NoOperatorReuseCheckerHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/NoOperatorReuseCheckerHook.java index 9a6a2e9..097cb4e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/NoOperatorReuseCheckerHook.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/NoOperatorReuseCheckerHook.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.hooks; -import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index 5c4f2a2..b4dac43 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -95,6 +95,7 @@ static final private String CLASS_NAME = DbTxnManager.class.getName(); static final private Logger LOG = LoggerFactory.getLogger(CLASS_NAME); + public static final String GLOBAL_LOCKS = "__GLOBAL_LOCKS"; private volatile DbLockManager lockMgr = null; /** @@ -449,7 +450,7 @@ LockComponentBuilder compBuilder = new LockComponentBuilder(); compBuilder.setExclusive(); compBuilder.setOperationType(DataOperationType.UPDATE); - compBuilder.setDbName("__GLOBAL_LOCKS"); + compBuilder.setDbName(GLOBAL_LOCKS); compBuilder.setTableName(lockName); globalLocks.add(compBuilder.build()); LOG.debug("Adding global lock: " + lockName); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ScheduledQueryAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ScheduledQueryAnalyzer.java index 66394a3..96b62de 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ScheduledQueryAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ScheduledQueryAnalyzer.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.ScheduledQueryMaintenanceRequestType; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.scheduled.ScheduledQueryMaintenanceWork; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; @@ -64,6 +65,7 @@ } work = new ScheduledQueryMaintenanceWork(type, schq); rootTasks.add(TaskFactory.get(work)); + queryState.setCommandType(toHiveOperation(type)); } private ScheduledQuery fillScheduledQuery(ScheduledQueryMaintenanceRequestType type, ScheduledQuery schqChanges) @@ -216,6 +218,19 @@ } } + private HiveOperation toHiveOperation(ScheduledQueryMaintenanceRequestType type) throws SemanticException { + switch (type) { + case CREATE: + return HiveOperation.CREATE_SCHEDULED_QUERY; + case ALTER: + return HiveOperation.ALTER_SCHEDULED_QUERY; + case DROP: + return HiveOperation.DROP_SCHEDULED_QUERY; + default: + throw new SemanticException("Unexpected type: " + type); + } + } + /** * Unparses the input AST node into correctly quoted sql string. */ diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java index 02fc69d..4ceafb0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java @@ -140,7 +140,7 @@ * @param node * source node (which must be an tabName) to be replaced */ - void addTableNameTranslation(ASTNode tableName, String currentDatabaseName) { + public void addTableNameTranslation(ASTNode tableName, String currentDatabaseName) { if (!enabled) { return; } @@ -221,7 +221,7 @@ */ void applyTranslations(TokenRewriteStream tokenRewriteStream) { for (Map.Entry entry : translations.entrySet()) { - if (entry.getKey() > 0) { // negative means the key didn't exist in the original + if (entry.getKey() > 0) { // negative means the key didn't exist in the original // stream (i.e.: we changed the tree) tokenRewriteStream.replace( entry.getKey(), diff --git ql/src/test/queries/clientpositive/schq_materialized.q ql/src/test/queries/clientpositive/schq_materialized.q new file mode 100644 index 0000000..bf67c95 --- /dev/null +++ ql/src/test/queries/clientpositive/schq_materialized.q @@ -0,0 +1,55 @@ +--! qt:sysdb + +drop materialized view if exists mv1; +drop table if exists emps; +drop table if exists depts; + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +-- create some tables +CREATE TABLE emps ( + empid INT, + deptno INT, + name VARCHAR(256), + salary FLOAT, + hire_date TIMESTAMP) +STORED AS ORC +TBLPROPERTIES ('transactional'='true'); + +CREATE TABLE depts ( + deptno INT, + deptname VARCHAR(256), + locationid INT) +STORED AS ORC +TBLPROPERTIES ('transactional'='true'); + +-- load data +insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (120, 10, 'Bill', 10000, 250), + (1330, 10, 'Bill', 10000, '2020-01-02'); +insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); + + +insert into emps values (1330, 10, 'Bill', 10000, '2020-01-02'); + +-- create mv +CREATE MATERIALIZED VIEW mv1 AS + SELECT empid, deptname, hire_date FROM emps + JOIN depts ON (emps.deptno = depts.deptno) + WHERE hire_date >= '2016-01-01 00:00:00'; + +EXPLAIN +SELECT empid, deptname FROM emps +JOIN depts ON (emps.deptno = depts.deptno) +WHERE hire_date >= '2018-01-01'; + +-- create a schedule to rebuild mv +create scheduled query d cron '0 */10 * * * ? *' defined as + alter materialized view mv1 rebuild; + +-- expected result to have it created +select * from sys.scheduled_queries; diff --git ql/src/test/results/clientpositive/llap/schq_materialized.q.out ql/src/test/results/clientpositive/llap/schq_materialized.q.out new file mode 100644 index 0000000..622cd5b --- /dev/null +++ ql/src/test/results/clientpositive/llap/schq_materialized.q.out @@ -0,0 +1,162 @@ +PREHOOK: query: drop materialized view if exists mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: drop materialized view if exists mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: drop table if exists emps +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists emps +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists depts +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists depts +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE emps ( + empid INT, + deptno INT, + name VARCHAR(256), + salary FLOAT, + hire_date TIMESTAMP) +STORED AS ORC +TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: CREATE TABLE emps ( + empid INT, + deptno INT, + name VARCHAR(256), + salary FLOAT, + hire_date TIMESTAMP) +STORED AS ORC +TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: CREATE TABLE depts ( + deptno INT, + deptname VARCHAR(256), + locationid INT) +STORED AS ORC +TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: CREATE TABLE depts ( + deptno INT, + deptname VARCHAR(256), + locationid INT) +STORED AS ORC +TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (120, 10, 'Bill', 10000, 250), + (1330, 10, 'Bill', 10000, '2020-01-02') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (120, 10, 'Bill', 10000, 250), + (1330, 10, 'Bill', 10000, '2020-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.hire_date SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptname SCRIPT [] +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +PREHOOK: query: insert into emps values (1330, 10, 'Bill', 10000, '2020-01-02') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (1330, 10, 'Bill', 10000, '2020-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.hire_date SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: CREATE MATERIALIZED VIEW mv1 AS + SELECT empid, deptname, hire_date FROM emps + JOIN depts ON (emps.deptno = depts.deptno) + WHERE hire_date >= '2016-01-01 00:00:00' +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: CREATE MATERIALIZED VIEW mv1 AS + SELECT empid, deptname, hire_date FROM emps + JOIN depts ON (emps.deptno = depts.deptno) + WHERE hire_date >= '2016-01-01 00:00:00' +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: EXPLAIN +SELECT empid, deptname FROM emps +JOIN depts ON (emps.deptno = depts.deptno) +WHERE hire_date >= '2018-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT empid, deptname FROM emps +JOIN depts ON (emps.deptno = depts.deptno) +WHERE hire_date >= '2018-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1 + filterExpr: (hire_date >= TIMESTAMP'2018-01-01 00:00:00') (type: boolean) + Filter Operator + predicate: (hire_date >= TIMESTAMP'2018-01-01 00:00:00') (type: boolean) + Select Operator + expressions: empid (type: int), deptname (type: varchar(256)) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: create scheduled query d cron '0 */10 * * * ? *' defined as + alter materialized view mv1 rebuild +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query d cron '0 */10 * * * ? *' defined as + alter materialized view mv1 rebuild +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: select * from sys.scheduled_queries +PREHOOK: type: QUERY +PREHOOK: Input: sys@scheduled_queries +#### A masked pattern was here #### +POSTHOOK: query: select * from sys.scheduled_queries +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@scheduled_queries +#### A masked pattern was here #### +1 d true hive 0 */10 * * * ? * hive_test_user alter materialized view `default`.`mv1` rebuild 1580503800