diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 4e0c50a662..786dacaddf 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -23,6 +23,9 @@ public static final String LLAP_LOGGER_NAME_CONSOLE = "console"; public static final String LLAP_LOGGER_NAME_RFA = "RFA"; + /* Constants for MV */ + public static final String MATERIALIZED_VIEW_VERSION = "materialized.view.version"; + /* Constants for Druid storage handler */ public static final String DRUID_HIVE_STORAGE_HANDLER_ID = "org.apache.hadoop.hive.druid.DruidStorageHandler"; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java index 53d716bceb..9ed231a584 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HooksLoader; -import org.apache.hadoop.hive.ql.hooks.MaterializedViewRegistryUpdateHook; import org.apache.hadoop.hive.ql.hooks.MetricsQueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContext; @@ -60,7 +59,6 @@ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { queryHooks.add(new MetricsQueryLifeTimeHook()); } - queryHooks.add(new MaterializedViewRegistryUpdateHook()); List propertyDefinedHoooks; try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 96fc33016b..9d46b36d79 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -23,6 +23,7 @@ import java.io.BufferedWriter; import java.io.DataOutputStream; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStreamWriter; @@ -237,6 +238,7 @@ import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; import org.apache.hadoop.hive.ql.plan.ShowTblPropertiesDesc; import org.apache.hadoop.hive.ql.plan.ShowTxnsDesc; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc; import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.hive.ql.plan.TruncateTableDesc; @@ -5119,13 +5121,26 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { } if (crtView.isMaterialized()) { + // Recall that this is a REBUILD // We need to update the status of the creation signature String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY); oldview.getTTable().setCreationMetadata( generateCreationMetadata(db, crtView.getTablesUsed(), txnString == null ? null : new ValidReadTxnList(txnString))); + // We disable the stats for the time being + oldview.getTTable().getParameters().remove(StatsSetupConst.COLUMN_STATS_ACCURATE); + // We need to set the properties and location so we get the new version + oldview.getTTable().getParameters().put(Constants.MATERIALIZED_VIEW_VERSION, + crtView.getTblProps().get(Constants.MATERIALIZED_VIEW_VERSION)); + final Path prevDataLocation = oldview.getDataLocation(); + oldview.getTTable().getSd().setLocation(crtView.getLocation()); + // We update metastore db.alterTable(crtView.getViewName(), oldview, null); - // This is a replace/rebuild, so we need an exclusive lock + // As table object is modified in this method, we need to update + // the subsequent stats tasks (if any) + updateChildrenStatsTask(oldview); + // We need to delete the previous location for the materialized view + deleteDir(prevDataLocation); addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE)); } else { // replace existing view @@ -5148,7 +5163,7 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { } oldview.checkValidity(null); db.alterTable(crtView.getViewName(), oldview, null); - addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); + addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_SHARED)); } } else { // We create new view @@ -5161,6 +5176,9 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { txnString == null ? null : new ValidReadTxnList(txnString))); } db.createTable(tbl, crtView.getIfNotExists()); + // As table object is modified in this method, we need to update + // the subsequent stats tasks (if any) + updateChildrenStatsTask(tbl); addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); //set lineage info @@ -5202,6 +5220,19 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { return signature; } + private void updateChildrenStatsTask(Table viewTbl) { + // As table object is modified in this method, we need to update + // the subsequent stats tasks (if any) + if (getChildTasks() != null) { + for (Task t : getChildTasks()) { + if (t.getWork() instanceof StatsWork) { + StatsWork sw = (StatsWork) t.getWork(); + sw.setTable(viewTbl); + } + } + } + } + private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws HiveException { if (truncateTableDesc.getColumnIndexes() != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryTask.java new file mode 100644 index 0000000000..b4abef9c20 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryTask.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; +import org.apache.hadoop.hive.ql.plan.api.StageType; + +import java.io.Serializable; + +/** + * This task adds the materialized view to the registry. + */ +public class MaterializedViewUpdateRegistryTask extends Task implements Serializable { + + private static final long serialVersionUID = 1L; + + public MaterializedViewUpdateRegistryTask() { + super(); + } + + @Override + public int execute(DriverContext driverContext) { + if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { + return 0; + } + try { + if (getWork().isRetrieveAndInclude()) { + Hive db = Hive.get(conf); + Table mvTable = db.getTable(getWork().getViewName()); + HiveMaterializedViewsRegistry.get().createMaterializedView(db.getConf(), mvTable); + } else if (getWork().isDisableRewrite()) { + // Disabling rewriting, removing from cache + String[] names = getWork().getViewName().split("\\."); + HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0], names[1]); + } + } catch (HiveException e) { + LOG.debug("Exception during materialized view cache update", e); + } + return 0; + } + + @Override + public StageType getType() { + return StageType.DDL; + } + + @Override + public String getName() { + return MaterializedViewUpdateRegistryTask.class.getSimpleName(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryWork.java new file mode 100644 index 0000000000..35ae002e4a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewUpdateRegistryWork.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +import java.io.Serializable; + +@Explain(displayName = "Materialized View Registry Update", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class MaterializedViewUpdateRegistryWork implements Serializable { + private static final long serialVersionUID = 1L; + private final String viewName; + private final boolean retrieveAndInclude; + private final boolean disableRewrite; + + public MaterializedViewUpdateRegistryWork(String viewName, boolean retrieveAndInclude, boolean disableRewrite) { + this.viewName = viewName; + this.retrieveAndInclude = retrieveAndInclude; + this.disableRewrite = disableRewrite; + } + + public String getViewName() { + return viewName; + } + + public boolean isRetrieveAndInclude() { + return retrieveAndInclude; + } + + public boolean isDisableRewrite() { + return disableRewrite; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index 85cef86646..602b92ecdb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -83,6 +83,9 @@ public TaskTuple(Class workClass, Class> taskClass) { taskvec.add(new TaskTuple(CopyWork.class, CopyTask.class)); taskvec.add(new TaskTuple(ReplCopyWork.class, ReplCopyTask.class)); taskvec.add(new TaskTuple(DDLWork.class, DDLTask.class)); + taskvec.add(new TaskTuple( + MaterializedViewUpdateRegistryWork.class, + MaterializedViewUpdateRegistryTask.class)); taskvec.add(new TaskTuple(FunctionWork.class, FunctionTask.class)); taskvec diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java deleted file mode 100644 index e886399d53..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MaterializedViewRegistryUpdateHook.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.hooks; - -import java.io.Serializable; -import java.util.List; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.exec.DDLTask; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.TaskRunner; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.plan.DDLWork; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Updates the materialized view registry after changes. - */ -public class MaterializedViewRegistryUpdateHook implements QueryLifeTimeHook { - - private static final Logger LOG = LoggerFactory.getLogger(MaterializedViewRegistryUpdateHook.class); - - @Override - public void beforeCompile(QueryLifeTimeHookContext ctx) { - } - - @Override - public void afterCompile(QueryLifeTimeHookContext ctx, boolean hasError) { - } - - @Override - public void beforeExecution(QueryLifeTimeHookContext ctx) { - } - - @Override - public void afterExecution(QueryLifeTimeHookContext ctx, boolean hasError) { - if (hasError) { - return; - } - HiveConf hiveConf = ctx.getHiveConf(); - try { - List completedTasks = ctx.getHookContext().getCompleteTaskList(); - for (TaskRunner taskRunner : completedTasks) { - Task task = taskRunner.getTask(); - if (task instanceof DDLTask) { - DDLTask ddlTask = (DDLTask) task; - DDLWork work = ddlTask.getWork(); - String tableName = null; - boolean isRewriteEnabled = false; - if (work.getCreateViewDesc() != null && work.getCreateViewDesc().isMaterialized()) { - tableName = work.getCreateViewDesc().toTable(hiveConf).getFullyQualifiedName(); - isRewriteEnabled = work.getCreateViewDesc().isRewriteEnabled(); - } else if (work.getAlterMaterializedViewDesc() != null) { - tableName = work.getAlterMaterializedViewDesc().getMaterializedViewName(); - isRewriteEnabled = work.getAlterMaterializedViewDesc().isRewriteEnable(); - } else { - continue; - } - - if (isRewriteEnabled) { - Hive db = Hive.get(); - Table mvTable = db.getTable(tableName); - HiveMaterializedViewsRegistry.get().createMaterializedView(db.getConf(), mvTable); - } else if (work.getAlterMaterializedViewDesc() != null) { - // Disabling rewriting, removing from cache - String[] names = tableName.split("\\."); - HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0], names[1]); - } - } - } - } catch (HiveException e) { - if (HiveConf.getBoolVar(hiveConf, ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING)) { - String message = "Error updating materialized view cache; consider disabling: " + ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING.varname; - LOG.error(message, e); - throw new RuntimeException(message, e); - } else { - LOG.debug("Exception during materialized view cache update", e); - } - } - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 923372d5b6..6f553154db 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -87,6 +87,7 @@ import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles.ConditionalResolverMergeFilesCtx; import org.apache.hadoop.hive.ql.plan.ConditionalWork; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -1508,25 +1509,7 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, LOG.debug("can't pre-create table for CTAS", e); table = null; } - } else if (mvWork.getLoadFileWork().getCreateViewDesc() != null) { - if (mvWork.getLoadFileWork().getCreateViewDesc().isReplace()) { - // ALTER MV ... REBUILD - String tableName = mvWork.getLoadFileWork().getCreateViewDesc().getViewName(); - try { - table = Hive.get().getTable(tableName); - } catch (HiveException e) { - throw new RuntimeException("unexpected; MV should be present already..: " + tableName, e); - } - } else { - // CREATE MATERIALIZED VIEW ... - try { - table = mvWork.getLoadFileWork().getCreateViewDesc().toTable(hconf); - } catch (HiveException e) { - LOG.debug("can't pre-create table for MV", e); - table = null; - } - } - } else { + } else if (mvWork.getLoadFileWork().getCreateViewDesc() == null) { throw new RuntimeException("unexpected; this should be a CTAS or a CREATE/REBUILD MV - however no desc present"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d0fe8ab322..5c5e6b9ae3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -45,7 +45,6 @@ import org.antlr.runtime.ClassicToken; import org.antlr.runtime.CommonToken; -import org.antlr.runtime.Token; import org.antlr.runtime.TokenRewriteStream; import org.antlr.runtime.tree.Tree; import org.antlr.runtime.tree.TreeVisitor; @@ -63,6 +62,7 @@ import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StatsSetupConst.StatDB; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; @@ -72,7 +72,6 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; @@ -12283,7 +12282,7 @@ ASTNode analyzeCreateTable( } } - addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE); + addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, WriteEntity.WriteType.DDL_NO_LOCK); if (isTemporary) { if (partCols.size() > 0) { @@ -12429,13 +12428,13 @@ ASTNode analyzeCreateTable( return null; } - private void addDbAndTabToOutputs(String[] qualifiedTabName, TableType type) throws SemanticException { + private void addDbAndTabToOutputs(String[] qualifiedTabName, TableType type, WriteEntity.WriteType writeType) throws SemanticException { Database database = getDatabase(qualifiedTabName[0]); outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED)); Table t = new Table(qualifiedTabName[0], qualifiedTabName[1]); t.setTableType(type); - outputs.add(new WriteEntity(t, WriteEntity.WriteType.DDL_NO_LOCK)); + outputs.add(new WriteEntity(t, writeType)); } protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCtx) throws SemanticException { @@ -12539,8 +12538,62 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, storageFormat.getSerde(), storageFormat.getStorageHandler(), storageFormat.getSerdeProps()); - addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW); queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); + + // For materialized views, properties should exist + if (createVwDesc.getTblProps() == null) { + createVwDesc.setTblProps(new HashMap<>()); + } + + Path dataLocation; + String mvVersion; + if (isRebuild) { + // We need to go lookup the table and get the select statement and then parse it. + Table tab; + try { + tab = getTableObjectByName(dbDotTable, true); + // We need to use the expanded text for the materialized view, as it will contain + // the qualified table aliases, etc. + String viewText = tab.getViewExpandedText(); + if (viewText.trim().isEmpty()) { + throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY); + } + Context ctx = new Context(queryState.getConf()); + selectStmt = ParseUtils.parse(viewText, ctx); + // For CBO + if (plannerCtx != null) { + plannerCtx.setViewToken(selectStmt); + } + } catch (Exception e) { + throw new SemanticException(e); + } + // Generate the new directory and increase the version + dataLocation = tab.getDataLocation().getParent(); + mvVersion = String.valueOf(Integer.parseInt( + tab.getProperty(Constants.MATERIALIZED_VIEW_VERSION)) + 1); + // Rebuild materialized view, exclusive lock + addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW, WriteEntity.WriteType.DDL_EXCLUSIVE); + outputs.add(BaseSemanticAnalyzer.toWriteEntity(tab.getDataLocation(), conf)); + } else { + // Add version property ('0') and set up location correctly + if (createVwDesc.getLocation() == null) { + try { + dataLocation = new Warehouse(conf).getDefaultTablePath( + db.getDatabase(qualTabName[0]), qualTabName[1]); + } catch (Exception e) { + throw new SemanticException(e); + } + } else { + dataLocation = new Path(createVwDesc.getLocation()); + } + // We create a new materialized view, hence we use version 0 + mvVersion = String.valueOf(0); + // New materialized view, no need for lock + addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW, WriteEntity.WriteType.DDL_NO_LOCK); + } + // Set up the new directory and version in tblProps + createVwDesc.setLocation(new Path(dataLocation, mvVersion).toString()); + createVwDesc.getTblProps().put(Constants.MATERIALIZED_VIEW_VERSION, mvVersion); } else { createVwDesc = new CreateViewDesc( dbDotTable, cols, comment, tblProps, partColNames, @@ -12548,32 +12601,11 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt storageFormat.getOutputFormat(), storageFormat.getSerde()); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), createVwDesc), conf)); - addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); + addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW, WriteEntity.WriteType.DDL_NO_LOCK); queryState.setCommandType(HiveOperation.CREATEVIEW); } qb.setViewDesc(createVwDesc); - if (isRebuild) { - // We need to go lookup the table and get the select statement and then parse it. - try { - Table tab = getTableObjectByName(dbDotTable, true); - // We need to use the expanded text for the materialized view, as it will contain - // the qualified table aliases, etc. - String viewText = tab.getViewExpandedText(); - if (viewText.trim().isEmpty()) { - throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY); - } - Context ctx = new Context(queryState.getConf()); - selectStmt = ParseUtils.parse(viewText, ctx); - // For CBO - if (plannerCtx != null) { - plannerCtx.setViewToken(selectStmt); - } - } catch (Exception e) { - throw new SemanticException(e); - } - } - return selectStmt; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 0c1c4e09d6..3148962798 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -28,6 +28,8 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.hive.ql.exec.DDLTask; +import org.apache.hadoop.hive.ql.exec.MaterializedViewUpdateRegistryWork; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +74,6 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.session.LineageState; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde.serdeConstants; @@ -490,6 +491,7 @@ private void patchUpAfterCTASorMaterializedView(final List> leaves = new LinkedHashSet<>(); getLeafTasks(rootTasks, leaves); assert (leaves.size() > 0); + Task targetTask = createTask; for (Task task : leaves) { if (task instanceof StatsTask) { // StatsTask require table to already exist @@ -500,10 +502,36 @@ private void patchUpAfterCTASorMaterializedView(final List 10.0 @@ -198,6 +201,9 @@ STAGE PLANS: Stats Work Basic Stats Work: + Stage: Stage-6 + Materialized View Registry Update + PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS SELECT cmv_basetable.a, cmv_basetable_2.c FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -443,6 +449,7 @@ STAGE DEPENDENCIES: Stage-0 depends on stages: Stage-2 Stage-5 depends on stages: Stage-0 Stage-3 depends on stages: Stage-5 + Stage-6 depends on stages: Stage-3 STAGE PLANS: Stage: Stage-1 @@ -534,6 +541,8 @@ STAGE PLANS: Create View Operator: Create View columns: a int, c decimal(10,2) + table properties: + materialized.view.version 1 name: default.cmv_mat_view replace: true @@ -541,6 +550,9 @@ STAGE PLANS: Stats Work Basic Stats Work: + Stage: Stage-6 + Materialized View Registry Update + PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: CREATE_MATERIALIZED_VIEW PREHOOK: Input: default@cmv_basetable diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out index 8ab1517186..68a82a8481 100644 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out @@ -92,6 +92,7 @@ STAGE DEPENDENCIES: Stage-0 depends on stages: Stage-2 Stage-5 depends on stages: Stage-0 Stage-3 depends on stages: Stage-5 + Stage-6 depends on stages: Stage-3 STAGE PLANS: Stage: Stage-1 @@ -183,6 +184,8 @@ STAGE PLANS: Create View Operator: Create View columns: a int, c decimal(10,2) + table properties: + materialized.view.version 0 expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) WHERE `cmv_basetable_2`.`c` > 10.0 @@ -197,6 +200,9 @@ STAGE PLANS: Stats Work Basic Stats Work: + Stage: Stage-6 + Materialized View Registry Update + PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view AS SELECT cmv_basetable.a, cmv_basetable_2.c FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -365,6 +371,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Output: default@cmv_basetable_2 #### A masked pattern was here #### +Found 1 items +#### A masked pattern was here #### PREHOOK: query: EXPLAIN ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE @@ -389,6 +397,8 @@ POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE POSTHOOK: Input: default@cmv_mat_view POSTHOOK: Output: default@cmv_mat_view +Found 1 items +#### A masked pattern was here #### PREHOOK: query: EXPLAIN SELECT cmv_basetable.a FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -526,6 +536,7 @@ STAGE DEPENDENCIES: Stage-0 depends on stages: Stage-2 Stage-5 depends on stages: Stage-0 Stage-3 depends on stages: Stage-5 + Stage-6 depends on stages: Stage-3 STAGE PLANS: Stage: Stage-1 @@ -617,6 +628,8 @@ STAGE PLANS: Create View Operator: Create View columns: a int, c decimal(10,2) + table properties: + materialized.view.version 1 name: default.cmv_mat_view replace: true @@ -624,6 +637,9 @@ STAGE PLANS: Stats Work Basic Stats Work: + Stage: Stage-6 + Materialized View Registry Update + PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: CREATE_MATERIALIZED_VIEW PREHOOK: Input: default@cmv_basetable @@ -636,6 +652,8 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Output: database:default POSTHOOK: Output: default@cmv_mat_view +Found 1 items +#### A masked pattern was here #### PREHOOK: query: EXPLAIN SELECT cmv_basetable.a FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out index d7ee468b49..6dee5ec48b 100644 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out +++ b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out @@ -89,6 +89,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +materialized.view.version 0 numFiles 1 numRows 2 rawDataSize 408 @@ -121,6 +122,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +materialized.view.version 0 numFiles 1 numRows 2 rawDataSize 232 diff --git a/ql/src/test/results/clientpositive/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/materialized_view_describe.q.out index efd60d4345..fb96fef4e0 100644 --- a/ql/src/test/results/clientpositive/materialized_view_describe.q.out +++ b/ql/src/test/results/clientpositive/materialized_view_describe.q.out @@ -69,6 +69,7 @@ Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} comment this is the first view key foo + materialized.view.version 0 numFiles 1 numRows 5 rawDataSize 580 @@ -95,6 +96,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} comment this is the first view key foo +materialized.view.version 0 numFiles 1 numRows 5 rawDataSize 580 @@ -157,6 +159,7 @@ Table Parameters: comment this is the second view key alice key2 bob + materialized.view.version 0 numFiles 1 numRows 5 rawDataSize 5 @@ -235,6 +238,7 @@ Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} comment this is the third view + materialized.view.version 0 numFiles 1 numRows 5 rawDataSize 1025 @@ -324,6 +328,7 @@ Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} comment this is the last view + materialized.view.version 0 numFiles 1 numRows 5 rawDataSize 5