diff --git parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index b03b0989b8..0178bd9112 100644 --- parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -267,6 +267,7 @@ TOK_ALTERVIEW_DROPPROPERTIES; TOK_ALTERVIEW_ADDPARTS; TOK_ALTERVIEW_DROPPARTS; TOK_ALTERVIEW_RENAME; +TOK_ALTERVIEW_AS; TOK_CREATE_MATERIALIZED_VIEW; TOK_DROP_MATERIALIZED_VIEW; TOK_ALTER_MATERIALIZED_VIEW; @@ -1257,7 +1258,7 @@ alterViewStatementSuffix | alterStatementSuffixRename[false] | alterStatementSuffixAddPartitions[false] | alterStatementSuffixDropPartitions[false] - | selectStatementWithCTE + | selectStatementWithCTE -> ^(TOK_ALTERVIEW_AS selectStatementWithCTE) ; alterMaterializedViewStatementSuffix[CommonTree tableNameTree] diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java index b82fc5e91d..0aea302524 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java @@ -32,8 +32,11 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.Entity.Type; @@ -237,4 +240,15 @@ public static String getFQName(ASTNode node) { return null; } } + + public static void addDbAndTableToOutputs(Database database, TableName tableName, TableType type, boolean isTemporary, + Map properties, Set outputs) { + outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED)); + + Table table = new Table(tableName.getDb(), tableName.getTable()); + table.setParameters(properties); + table.setTableType(type); + table.setTemporary(isTemporary); + outputs.add(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK)); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java new file mode 100644 index 0000000000..4b20421122 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.PlanUtils; + +/** + * Abstract ancestor of analyzers that can create a view. + */ +abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer { + public AbstractCreateViewAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + } + + protected SemanticAnalyzer analyzeQuery(ASTNode select, String viewName) throws SemanticException { + QueryState innerQueryState = new QueryState.Builder().withHiveConf(conf).build(); + innerQueryState.getConf().setBoolVar(HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES, false); + + SemanticAnalyzer analyzer = (SemanticAnalyzer) SemanticAnalyzerFactory.get(innerQueryState, select); + ctx.setEnableUnparse(true); + analyzer.setForViewCreation(true); + analyzer.analyze(select, ctx); + analyzer.executeUnparseTranlations(); + + queryState.setLineageState(innerQueryState.getLineageState()); + queryState.getLineageState().mapDirToOp(new Path(viewName), analyzer.getSinkOp()); + + addInputs(analyzer); + + return analyzer; + } + + private void addInputs(SemanticAnalyzer analyzer) { + inputs.addAll(analyzer.getInputs()); + for (Map.Entry entry : analyzer.getTopOps().entrySet()) { + String alias = entry.getKey(); + TableScanOperator topOp = entry.getValue(); + ReadEntity parentViewInfo = PlanUtils.getParentViewInfo(alias, analyzer.getViewAliasToInput()); + + // Adds tables only for create view (PPD filter can be appended by outer query) + Table table = topOp.getConf().getTableMetadata(); + PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo)); + } + } + + protected void validateTablesUsed(SemanticAnalyzer analyzer) throws SemanticException { + // Do not allow view to be defined on temp table or other materialized view + Set tableAliases = analyzer.getQB().getTabAliases(); + for (String alias : tableAliases) { + try { + if (SemanticAnalyzer.DUMMY_TABLE.equals(alias)) { + continue; + } + Table table = analyzer.getTableObjectByName(analyzer.getQB().getTabNameForAlias(alias)); + if (table.isTemporary()) { + throw new SemanticException("View definition references temporary table " + alias); + } + if (table.isMaterializedView()) { + throw new SemanticException("View definition references materialized view " + alias); + } + } catch (HiveException ex) { + throw new SemanticException(ex); + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewDesc.java new file mode 100644 index 0000000000..eed65e9bfc --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewDesc.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.io.Serializable; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * Abstract ancestor of view creating DDL commands. + */ +abstract class AbstractCreateViewDesc implements DDLDesc, Serializable { + private static final long serialVersionUID = 1L; + + private final String viewName; + private final List schema; + private final String originalText; + private final String expandedText; + + /** + * Used to create a view descriptor. + */ + public AbstractCreateViewDesc(String viewName, List schema, String originalText, String expandedText) { + this.viewName = viewName; + this.schema = schema; + this.originalText = originalText; + this.expandedText = expandedText; + } + + @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getViewName() { + return viewName; + } + + @Explain(displayName = "original text", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getOriginalText() { + return originalText; + } + + @Explain(displayName = "expanded text") + public String getExpandedText() { + return expandedText; + } + + @Explain(displayName = "columns") + public List getSchemaString() { + return Utilities.getFieldSchemaString(schema); + } + + public List getSchema() { + return schema; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsAnalyzer.java new file mode 100644 index 0000000000..2ef0433704 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsAnalyzer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Analyzer for alter view ... as commands. + */ +@DDLType(types = HiveParser.TOK_ALTERVIEW_AS) +public class AlterViewAsAnalyzer extends AbstractCreateViewAnalyzer { + public AlterViewAsAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + } + + @Override + public void analyzeInternal(ASTNode root) throws SemanticException { + TableName viewName = getQualifiedTableName((ASTNode) root.getChild(0)); + String fqViewName = viewName.getNotEmptyDbTable(); + LOG.info("Altering the query of view " + fqViewName + " position=" + root.getCharPositionInLine()); + + ASTNode select = (ASTNode) root.getChild(1).getChild(0); + + String originalText = ctx.getTokenRewriteStream().toString(select.getTokenStartIndex(), select.getTokenStopIndex()); + + SemanticAnalyzer analyzer = analyzeQuery(select, fqViewName); + + List querySchema = new ArrayList(analyzer.getResultSchema()); + ParseUtils.validateColumnNameUniqueness(querySchema); + + String expandedText = ctx.getTokenRewriteStream().toString(select.getTokenStartIndex(), select.getTokenStopIndex()); + + AlterViewAsDesc desc = new AlterViewAsDesc(fqViewName, querySchema, originalText, expandedText); + validateCreateView(desc, analyzer); + + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + DDLUtils.addDbAndTableToOutputs(getDatabase(viewName.getDb()), viewName, TableType.VIRTUAL_VIEW, false, + null, outputs); + } + + private void validateCreateView(AlterViewAsDesc desc, SemanticAnalyzer analyzer) throws SemanticException { + try { + validateTablesUsed(analyzer); + + //replace view + Table oldView = getTable(desc.getViewName(), false); + if (oldView == null) { + String viewNotExistErrorMsg = "The following view does not exist: " + desc.getViewName(); + throw new SemanticException( + ErrorMsg.ALTER_VIEW_AS_SELECT_NOT_EXIST.getMsg(viewNotExistErrorMsg)); + } + } catch (HiveException e) { + throw new SemanticException(e.getMessage(), e); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsDesc.java new file mode 100644 index 0000000000..0a8e75720e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsDesc.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER VIEW ... AS ... commands. + */ +@Explain(displayName = "Alter View As", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterViewAsDesc extends AbstractCreateViewDesc { + private static final long serialVersionUID = 1L; + + public AlterViewAsDesc(String viewName, List schema, String originalText, String expandedText) { + super(viewName, schema, originalText, expandedText); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsOperation.java new file mode 100644 index 0000000000..d7e26a235f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AlterViewAsOperation.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of altering a view with a new query. + */ +public class AlterViewAsOperation extends DDLOperation { + public AlterViewAsOperation(DDLOperationContext context, AlterViewAsDesc desc) { + super(context, desc); + } + + @Override + public int execute() throws HiveException { + Table oldview = context.getDb().getTable(desc.getViewName(), false); + + // replace existing view + oldview.setViewOriginalText(desc.getOriginalText()); + oldview.setViewExpandedText(desc.getExpandedText()); + oldview.setFields(desc.getSchema()); + oldview.checkValidity(null); + + context.getDb().alterTable(desc.getViewName(), oldview, false, null, true); + DDLUtils.addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK), + context.getWork().getOutputs()); + + return 0; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java new file mode 100644 index 0000000000..529a755700 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java @@ -0,0 +1,417 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; +import org.apache.hadoop.hive.ql.plan.FileSinkDesc; +import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * DDL task description for CREATE VIEW commands. + */ +@Explain(displayName = "Create View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class CreateMaterializedViewDesc implements DDLDesc, Serializable { + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(CreateMaterializedViewDesc.class); + + private String viewName; + private List schema; + private String comment; + private Map tblProps; + private List partColNames; + private boolean ifNotExists; + + private String originalText; + private String expandedText; + private boolean rewriteEnabled; + private List partCols; + private String inputFormat; + private String outputFormat; + private String location; // only used for materialized views + private String serde; // only used for materialized views + private String storageHandler; // only used for materialized views + private Map serdeProps; // only used for materialized views + private Set tablesUsed; // only used for materialized views + private List sortColNames; // only used for materialized views + private List sortCols; // only used for materialized views + private List distributeColNames; // only used for materialized views + private List distributeCols; // only used for materialized views + private Long initialMmWriteId; // Initial MM write ID for CMV and import. + // The FSOP configuration for the FSOP that is going to write initial data during cmv. + // This is not needed beyond compilation, so it is transient. + private transient FileSinkDesc writer; + private String ownerName = null; + + /** + * Used to create a materialized view descriptor. + */ + public CreateMaterializedViewDesc(String viewName, List schema, String comment, Map tblProps, + List partColNames, List sortColNames, List distributeColNames, + boolean ifNotExists, boolean rewriteEnabled, + String inputFormat, String outputFormat, String location, + String serde, String storageHandler, Map serdeProps) { + this.viewName = viewName; + this.schema = schema; + this.comment = comment; + this.tblProps = tblProps; + this.partColNames = partColNames; + this.sortColNames = sortColNames; + this.distributeColNames = distributeColNames; + this.ifNotExists = ifNotExists; + + this.rewriteEnabled = rewriteEnabled; + this.inputFormat = inputFormat; + this.outputFormat = outputFormat; + this.location = location; + this.serde = serde; + this.storageHandler = storageHandler; + this.serdeProps = serdeProps; + } + + @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getViewName() { + return viewName; + } + + public void setViewName(String viewName) { + this.viewName = viewName; + } + + @Explain(displayName = "original text", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getViewOriginalText() { + return originalText; + } + + public void setViewOriginalText(String originalText) { + this.originalText = originalText; + } + + @Explain(displayName = "expanded text") + public String getViewExpandedText() { + return expandedText; + } + + public void setViewExpandedText(String expandedText) { + this.expandedText = expandedText; + } + + @Explain(displayName = "rewrite enabled", displayOnlyOnTrue = true) + public boolean isRewriteEnabled() { + return rewriteEnabled; + } + + public void setRewriteEnabled(boolean rewriteEnabled) { + this.rewriteEnabled = rewriteEnabled; + } + + @Explain(displayName = "columns") + public List getSchemaString() { + return Utilities.getFieldSchemaString(schema); + } + + public List getSchema() { + return schema; + } + + public void setSchema(List schema) { + this.schema = schema; + } + + @Explain(displayName = "partition columns") + public List getPartColsString() { + return Utilities.getFieldSchemaString(partCols); + } + + public List getPartCols() { + return partCols; + } + + public void setPartCols(List partCols) { + this.partCols = partCols; + } + + public List getPartColNames() { + return partColNames; + } + + public void setPartColNames(List partColNames) { + this.partColNames = partColNames; + } + + public boolean isOrganized() { + return (sortColNames != null && !sortColNames.isEmpty()) || + (distributeColNames != null && !distributeColNames.isEmpty()); + } + + @Explain(displayName = "sort columns") + public List getSortColsString() { + return Utilities.getFieldSchemaString(sortCols); + } + + public List getSortCols() { + return sortCols; + } + + public void setSortCols(List sortCols) { + this.sortCols = sortCols; + } + + public List getSortColNames() { + return sortColNames; + } + + public void setSortColNames(List sortColNames) { + this.sortColNames = sortColNames; + } + + @Explain(displayName = "distribute columns") + public List getDistributeColsString() { + return Utilities.getFieldSchemaString(distributeCols); + } + + public List getDistributeCols() { + return distributeCols; + } + + public void setDistributeCols(List distributeCols) { + this.distributeCols = distributeCols; + } + + public List getDistributeColNames() { + return distributeColNames; + } + + public void setDistributeColNames(List distributeColNames) { + this.distributeColNames = distributeColNames; + } + + @Explain(displayName = "comment") + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public void setTblProps(Map tblProps) { + this.tblProps = tblProps; + } + + @Explain(displayName = "table properties") + public Map getTblProps() { + return tblProps; + } + + @Explain(displayName = "if not exists", displayOnlyOnTrue = true) + public boolean getIfNotExists() { + return ifNotExists; + } + + public void setIfNotExists(boolean ifNotExists) { + this.ifNotExists = ifNotExists; + } + + public Set getTablesUsed() { + return tablesUsed; + } + + public void setTablesUsed(Set tablesUsed) { + this.tablesUsed = tablesUsed; + } + + public String getInputFormat() { + return inputFormat; + } + + public void setInputFormat(String inputFormat) { + this.inputFormat = inputFormat; + } + + public String getOutputFormat() { + return outputFormat; + } + + public void setOutputFormat(String outputFormat) { + this.outputFormat = outputFormat; + } + + public void setLocation(String location) { + this.location = location; + } + public String getLocation() { + return location; + } + + public String getSerde() { + return serde; + } + + public String getStorageHandler() { + return storageHandler; + } + + public Map getSerdeProps() { + return serdeProps; + } + + public Table toTable(HiveConf conf) throws HiveException { + String[] names = Utilities.getDbTableName(getViewName()); + String databaseName = names[0]; + String tableName = names[1]; + + Table tbl = new Table(databaseName, tableName); + tbl.setViewOriginalText(getViewOriginalText()); + tbl.setViewExpandedText(getViewExpandedText()); + tbl.setRewriteEnabled(isRewriteEnabled()); + tbl.setTableType(TableType.MATERIALIZED_VIEW); + tbl.setSerializationLib(null); + tbl.clearSerDeInfo(); + tbl.setFields(getSchema()); + if (getComment() != null) { + tbl.setProperty("comment", getComment()); + } + + if (tblProps != null) { + tbl.getParameters().putAll(tblProps); + } + + if (!CollectionUtils.isEmpty(partCols)) { + tbl.setPartCols(partCols); + } + + if (!CollectionUtils.isEmpty(sortColNames)) { + tbl.setProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS, + Utilities.encodeColumnNames(sortColNames)); + } + if (!CollectionUtils.isEmpty(distributeColNames)) { + tbl.setProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS, + Utilities.encodeColumnNames(distributeColNames)); + } + + if (getInputFormat() != null) { + tbl.setInputFormatClass(getInputFormat()); + } + + if (getOutputFormat() != null) { + tbl.setOutputFormatClass(getOutputFormat()); + } + + if (getLocation() != null) { + tbl.setDataLocation(new Path(getLocation())); + } + + if (getStorageHandler() != null) { + tbl.setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + getStorageHandler()); + } + HiveStorageHandler storageHandler = tbl.getStorageHandler(); + + /* + * If the user didn't specify a SerDe, we use the default. + */ + String serDeClassName; + if (getSerde() == null) { + if (storageHandler == null) { + serDeClassName = PlanUtils.getDefaultSerDe().getName(); + LOG.info("Default to {} for materialized view {}", serDeClassName, getViewName()); + } else { + serDeClassName = storageHandler.getSerDeClass().getName(); + LOG.info("Use StorageHandler-supplied {} for materialized view {}", serDeClassName, getViewName()); + } + } else { + // let's validate that the serde exists + serDeClassName = getSerde(); + DDLUtils.validateSerDe(serDeClassName, conf); + } + tbl.setSerializationLib(serDeClassName); + + // To remain consistent, we need to set input and output formats both + // at the table level and the storage handler level. + tbl.setInputFormatClass(getInputFormat()); + tbl.setOutputFormatClass(getOutputFormat()); + if (getInputFormat() != null && !getInputFormat().isEmpty()) { + tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName()); + } + if (getOutputFormat() != null && !getOutputFormat().isEmpty()) { + tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); + } + + if (ownerName != null) { + tbl.setOwner(ownerName); + } + + // Sets the column state for the create view statement (false since it is a creation). + // Similar to logic in CreateTableDesc. + StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null, + StatsSetupConst.FALSE); + + return tbl; + } + + public void setInitialMmWriteId(Long mmWriteId) { + this.initialMmWriteId = mmWriteId; + } + + public Long getInitialMmWriteId() { + return initialMmWriteId; + } + + public FileSinkDesc getAndUnsetWriter() { + FileSinkDesc fsd = writer; + writer = null; + return fsd; + } + + public void setWriter(FileSinkDesc writer) { + this.writer = writer; + } + + public void setOwnerName(String ownerName) { + this.ownerName = ownerName; + } + + public String getOwnerName() { + return this.ownerName; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewOperation.java new file mode 100644 index 0000000000..30ff28cae0 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewOperation.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; + +import com.google.common.collect.ImmutableSet; + +/** + * Operation process of creating a view. + */ +public class CreateMaterializedViewOperation extends DDLOperation { + public CreateMaterializedViewOperation(DDLOperationContext context, CreateMaterializedViewDesc desc) { + super(context, desc); + } + + @Override + public int execute() throws HiveException { + Table oldview = context.getDb().getTable(desc.getViewName(), false); + if (oldview != null) { + + if (desc.getIfNotExists()) { + return 0; + } + + // Materialized View already exists, thus we should be replacing + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName())); + } else { + // We create new view + Table tbl = desc.toTable(context.getConf()); + // We set the signature for the view if it is a materialized view + if (tbl.isMaterializedView()) { + CreationMetadata cm = + new CreationMetadata(MetaStoreUtils.getDefaultCatalog(context.getConf()), tbl.getDbName(), + tbl.getTableName(), ImmutableSet.copyOf(desc.getTablesUsed())); + cm.setValidTxnList(context.getConf().get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); + tbl.getTTable().setCreationMetadata(cm); + } + context.getDb().createTable(tbl, desc.getIfNotExists()); + DDLUtils.addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK), + context.getWork().getOutputs()); + + //set lineage info + DataContainer dc = new DataContainer(tbl.getTTable()); + context.getQueryState().getLineageState().setLineage(new Path(desc.getViewName()), dc, tbl.getCols()); + } + return 0; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java new file mode 100644 index 0000000000..7cd4df1836 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.view.create; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +/** + * Analyzer for create view commands. + */ +@DDLType(types = HiveParser.TOK_CREATEVIEW) +public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + } + + @Override + public void analyzeInternal(ASTNode root) throws SemanticException { + TableName viewName = getQualifiedTableName((ASTNode) root.getChild(0)); + String fqViewName = viewName.getNotEmptyDbTable(); + LOG.info("Creating view " + fqViewName + " position=" + root.getCharPositionInLine()); + + Map children = new HashMap<>(); + for (int i = 1; i < root.getChildCount(); i++) { + ASTNode child = (ASTNode) root.getChild(i); + children.put(child.getToken().getType(), child); + } + + List schema = children.containsKey(HiveParser.TOK_TABCOLNAME) ? + getColumns((ASTNode) children.remove(HiveParser.TOK_TABCOLNAME)) : null; + boolean ifNotExists = children.remove(HiveParser.TOK_IFNOTEXISTS) != null; + boolean orReplace = children.remove(HiveParser.TOK_ORREPLACE) != null; + String comment = children.containsKey(HiveParser.TOK_TABLECOMMENT) ? + unescapeSQLString(children.remove(HiveParser.TOK_TABLECOMMENT).getChild(0).getText()) : null; + ASTNode select = children.remove(HiveParser.TOK_QUERY); + Map properties = children.containsKey(HiveParser.TOK_TABLEPROPERTIES) ? + getProps((ASTNode) children.remove(HiveParser.TOK_TABLEPROPERTIES).getChild(0)) : null; + List partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? + getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; + + assert children.isEmpty(); + + if (ifNotExists && orReplace) { + throw new SemanticException("Can't combine IF NOT EXISTS and OR REPLACE."); + } + + String originalText = ctx.getTokenRewriteStream().toString(select.getTokenStartIndex(), select.getTokenStopIndex()); + + SemanticAnalyzer analyzer = analyzeQuery(select, fqViewName); + + List querySchema = new ArrayList(analyzer.getResultSchema()); + ParseUtils.validateColumnNameUniqueness(querySchema); + + String expandedText = getExpandedText(querySchema, schema, select, viewName); + + List partitionColumns = getPartitionColumns(querySchema, schema, select, viewName, + partitionColumnNames); + + CreateViewDesc desc = new CreateViewDesc(fqViewName, querySchema, comment, properties, partitionColumnNames, + ifNotExists, orReplace, originalText, expandedText, partitionColumns); + validateCreateView(desc, analyzer); + + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + DDLUtils.addDbAndTableToOutputs(getDatabase(viewName.getDb()), viewName, TableType.VIRTUAL_VIEW, false, + properties, outputs); + } + + private String getExpandedText(List querySchema, List imposedSchema, ASTNode select, + TableName viewName) throws SemanticException { + if (imposedSchema != null) { + if (imposedSchema.size() != querySchema.size()) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(select, ErrorMsg.VIEW_COL_MISMATCH.getMsg())); + } + } + + String expandedText = ctx.getTokenRewriteStream().toString(select.getTokenStartIndex(), select.getTokenStopIndex()); + + if (imposedSchema != null) { + // Merge the names from the imposed schema into the types from the derived schema. + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + for (int i = 0; i < querySchema.size(); i++) { + if (i > 0) { + sb.append(", "); + } + // Modify a copy, not the original + FieldSchema fieldSchema = new FieldSchema(querySchema.get(i)); + // TODO: there's a potential problem here if some table uses external schema like Avro, + // with a very large type name. It seems like the view does not derive the SerDe from + // the table, so it won't be able to just get the type from the deserializer like the + // table does; we won't be able to properly store the type in the RDBMS metastore. + // Not sure if these large cols could be in resultSchema. Ignore this for now 0_o + querySchema.set(i, fieldSchema); + sb.append(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)); + sb.append(" AS "); + String imposedName = imposedSchema.get(i).getName(); + sb.append(HiveUtils.unparseIdentifier(imposedName, conf)); + fieldSchema.setName(imposedName); + // We don't currently allow imposition of a type + fieldSchema.setComment(imposedSchema.get(i).getComment()); + } + sb.append(" FROM ("); + sb.append(expandedText); + sb.append(") "); + sb.append(HiveUtils.unparseIdentifier(viewName.getTable(), conf)); + + expandedText = sb.toString(); + } + return expandedText; + } + + private List getPartitionColumns(List querySchema, List imposedSchema, + ASTNode select, TableName viewName, List partitionColumnNames) throws SemanticException { + if (partitionColumnNames == null) { + return null; + } + + // Make sure all partitioning columns referenced actually exist and are in the correct order at the end of the + // list of columns produced by the view. Also move the field schema descriptors from derivedSchema to the + // partitioning key descriptor. + if (partitionColumnNames.size() > querySchema.size()) { + throw new SemanticException(ErrorMsg.VIEW_PARTITION_MISMATCH.getMsg()); + } + + // Boundary case: require at least one non-partitioned column for consistency with tables. + if (partitionColumnNames.size() == querySchema.size()) { + throw new SemanticException(ErrorMsg.VIEW_PARTITION_TOTAL.getMsg()); + } + + // Get the partition columns from the end of derivedSchema. + List partitionColumns = + querySchema.subList(querySchema.size() - partitionColumnNames.size(), querySchema.size()); + + // Verify that the names match the PARTITIONED ON clause. + Iterator columnNameIterator = partitionColumnNames.iterator(); + Iterator schemaIterator = partitionColumns.iterator(); + while (columnNameIterator.hasNext()) { + String columnName = columnNameIterator.next(); + FieldSchema fieldSchema = schemaIterator.next(); + if (!fieldSchema.getName().equals(columnName)) { + throw new SemanticException(ErrorMsg.VIEW_PARTITION_MISMATCH.getMsg()); + } + } + + // Now make a copy, and remove the partition columns from the end of derivedSchema. + // (Clearing the subList writes through to the underlying derivedSchema ArrayList.) + List partitionColumnsCopy = new ArrayList(partitionColumns); + partitionColumns.clear(); + return partitionColumnsCopy; + } + + private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { + try { + validateTablesUsed(analyzer); + + //replace view + Table oldView = getTable(desc.getViewName(), false); + if (desc.isReplace() && oldView != null) { + if (oldView.getTableType().equals(TableType.MATERIALIZED_VIEW)) { + throw new SemanticException(ErrorMsg.REPLACE_MATERIALIZED_WITH_VIEW, oldView.getTableName()); + } + + // Existing table is not a view + if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && + !oldView.getTableType().equals(TableType.MATERIALIZED_VIEW)) { + String tableNotViewErrorMsg = "The following is an existing table, not a view: " + desc.getViewName(); + throw new SemanticException(ErrorMsg.EXISTING_TABLE_IS_NOT_VIEW.getMsg(tableNotViewErrorMsg)); + } + + String partitionViewErrorMsg = "The following view has partition, it could not be replaced: " + + desc.getViewName(); + try { + if (!oldView.getPartCols().isEmpty() && !db.getPartitions(oldView).isEmpty() && + !oldView.getPartCols().equals(desc.getPartitionColumns())) { + throw new SemanticException(ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + } + } catch (HiveException e) { + throw new SemanticException(ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + } + } + } catch (HiveException e) { + throw new SemanticException(e.getMessage(), e); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index d1f36945fb..e71cbce773 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -18,238 +18,55 @@ package org.apache.hadoop.hive.ql.ddl.view.create; -import java.io.Serializable; import java.util.List; import java.util.Map; -import java.util.Set; -import org.apache.commons.collections.CollectionUtils; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.Constants; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.ddl.DDLDesc; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; -import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; -import org.apache.hadoop.hive.ql.plan.FileSinkDesc; -import org.apache.hadoop.hive.ql.plan.PlanUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * DDL task description for CREATE VIEW commands. */ @Explain(displayName = "Create View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class CreateViewDesc implements DDLDesc, Serializable { +public class CreateViewDesc extends AbstractCreateViewDesc { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(CreateViewDesc.class); - private String viewName; - private List schema; - private String comment; - private Map tblProps; - private List partColNames; - private boolean ifNotExists; - private boolean orReplace; + private final String comment; + private final Map properties; + private final List partitionColumnNames; + private final boolean ifNotExists; + private final boolean replace; + private final List partitionColumns; - private String originalText; - private String expandedText; - private boolean rewriteEnabled; - private List partCols; - private boolean isAlterViewAs; - private boolean isMaterialized; - private String inputFormat; - private String outputFormat; - private String location; // only used for materialized views - private String serde; // only used for materialized views - private String storageHandler; // only used for materialized views - private Map serdeProps; // only used for materialized views - private Set tablesUsed; // only used for materialized views - private List sortColNames; // only used for materialized views - private List sortCols; // only used for materialized views - private List distributeColNames; // only used for materialized views - private List distributeCols; // only used for materialized views private ReplicationSpec replicationSpec = null; - private Long initialMmWriteId; // Initial MM write ID for CMV and import. - // The FSOP configuration for the FSOP that is going to write initial data during cmv. - // This is not needed beyond compilation, so it is transient. - private transient FileSinkDesc writer; private String ownerName = null; - /** - * Used to create a materialized view descriptor. - */ - public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, - List partColNames, List sortColNames, List distributeColNames, - boolean ifNotExists, boolean orReplace, boolean rewriteEnabled, boolean isAlterViewAs, - String inputFormat, String outputFormat, String location, - String serde, String storageHandler, Map serdeProps) { - this.viewName = viewName; - this.schema = schema; + public CreateViewDesc(String viewName, List schema, String comment, Map properties, + List partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, + String expandedText, List partitionColumns) { + super(viewName, schema, originalText, expandedText); this.comment = comment; - this.tblProps = tblProps; - this.partColNames = partColNames; - this.sortColNames = sortColNames; - this.distributeColNames = distributeColNames; + this.properties = properties; + this.partitionColumnNames = partitionColumnNames; this.ifNotExists = ifNotExists; - this.orReplace = orReplace; - - this.isMaterialized = true; - this.rewriteEnabled = rewriteEnabled; - this.isAlterViewAs = isAlterViewAs; - this.inputFormat = inputFormat; - this.outputFormat = outputFormat; - this.location = location; - this.serde = serde; - this.storageHandler = storageHandler; - this.serdeProps = serdeProps; - } - - /** - * Used to create a view descriptor. - */ - public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, - List partColNames, boolean ifNotExists, boolean orReplace, - boolean isAlterViewAs, String inputFormat, String outputFormat, String serde) { - this.viewName = viewName; - this.schema = schema; - this.comment = comment; - this.tblProps = tblProps; - this.partColNames = partColNames; - this.ifNotExists = ifNotExists; - this.orReplace = orReplace; - - this.isAlterViewAs = isAlterViewAs; - this.isMaterialized = false; - this.rewriteEnabled = false; - this.inputFormat = inputFormat; - this.outputFormat = outputFormat; - this.serde = serde; - } - - @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getViewName() { - return viewName; - } - - public void setViewName(String viewName) { - this.viewName = viewName; - } - - @Explain(displayName = "original text", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getViewOriginalText() { - return originalText; - } - - public void setViewOriginalText(String originalText) { - this.originalText = originalText; - } - - @Explain(displayName = "expanded text") - public String getViewExpandedText() { - return expandedText; - } - - public void setViewExpandedText(String expandedText) { - this.expandedText = expandedText; - } - - @Explain(displayName = "rewrite enabled", displayOnlyOnTrue = true) - public boolean isRewriteEnabled() { - return rewriteEnabled; - } - - public void setRewriteEnabled(boolean rewriteEnabled) { - this.rewriteEnabled = rewriteEnabled; - } - - @Explain(displayName = "columns") - public List getSchemaString() { - return Utilities.getFieldSchemaString(schema); - } - - public List getSchema() { - return schema; - } - - public void setSchema(List schema) { - this.schema = schema; + this.replace = replace; + this.partitionColumns = partitionColumns; } @Explain(displayName = "partition columns") public List getPartColsString() { - return Utilities.getFieldSchemaString(partCols); + return Utilities.getFieldSchemaString(partitionColumns); } - public List getPartCols() { - return partCols; - } - - public void setPartCols(List partCols) { - this.partCols = partCols; + public List getPartitionColumns() { + return partitionColumns; } public List getPartColNames() { - return partColNames; - } - - public void setPartColNames(List partColNames) { - this.partColNames = partColNames; - } - - public boolean isOrganized() { - return (sortColNames != null && !sortColNames.isEmpty()) || - (distributeColNames != null && !distributeColNames.isEmpty()); - } - - @Explain(displayName = "sort columns") - public List getSortColsString() { - return Utilities.getFieldSchemaString(sortCols); - } - - public List getSortCols() { - return sortCols; - } - - public void setSortCols(List sortCols) { - this.sortCols = sortCols; - } - - public List getSortColNames() { - return sortColNames; - } - - public void setSortColNames(List sortColNames) { - this.sortColNames = sortColNames; - } - - @Explain(displayName = "distribute columns") - public List getDistributeColsString() { - return Utilities.getFieldSchemaString(distributeCols); - } - - public List getDistributeCols() { - return distributeCols; - } - - public void setDistributeCols(List distributeCols) { - this.distributeCols = distributeCols; - } - - public List getDistributeColNames() { - return distributeColNames; - } - - public void setDistributeColNames(List distributeColNames) { - this.distributeColNames = distributeColNames; + return partitionColumnNames; } @Explain(displayName = "comment") @@ -257,17 +74,9 @@ public String getComment() { return comment; } - public void setComment(String comment) { - this.comment = comment; - } - - public void setTblProps(Map tblProps) { - this.tblProps = tblProps; - } - - @Explain(displayName = "table properties") - public Map getTblProps() { - return tblProps; + @Explain(displayName = "properties") + public Map getProperties() { + return properties; } @Explain(displayName = "if not exists", displayOnlyOnTrue = true) @@ -275,73 +84,9 @@ public boolean getIfNotExists() { return ifNotExists; } - public void setIfNotExists(boolean ifNotExists) { - this.ifNotExists = ifNotExists; - } - - public Set getTablesUsed() { - return tablesUsed; - } - - public void setTablesUsed(Set tablesUsed) { - this.tablesUsed = tablesUsed; - } - @Explain(displayName = "replace", displayOnlyOnTrue = true) public boolean isReplace() { - return orReplace; - } - - public void setReplace(boolean replace) { - this.orReplace = replace; - } - - @Explain(displayName = "is alter view as select", displayOnlyOnTrue = true) - public boolean getIsAlterViewAs() { - return isAlterViewAs; - } - - public void setIsAlterViewAs(boolean isAlterViewAs) { - this.isAlterViewAs = isAlterViewAs; - } - - public String getInputFormat() { - return inputFormat; - } - - public void setInputFormat(String inputFormat) { - this.inputFormat = inputFormat; - } - - public String getOutputFormat() { - return outputFormat; - } - - public void setOutputFormat(String outputFormat) { - this.outputFormat = outputFormat; - } - - public boolean isMaterialized() { - return isMaterialized; - } - - public void setLocation(String location) { - this.location = location; - } - public String getLocation() { - return location; - } - - public String getSerde() { - return serde; - } - - public String getStorageHandler() { - return storageHandler; - } - - public Map getSerdeProps() { - return serdeProps; + return replace; } /** @@ -359,126 +104,7 @@ public ReplicationSpec getReplicationSpec() { if (replicationSpec == null) { this.replicationSpec = new ReplicationSpec(); } - return this.replicationSpec; - } - - public Table toTable(HiveConf conf) throws HiveException { - String[] names = Utilities.getDbTableName(getViewName()); - String databaseName = names[0]; - String tableName = names[1]; - - Table tbl = new Table(databaseName, tableName); - tbl.setViewOriginalText(getViewOriginalText()); - tbl.setViewExpandedText(getViewExpandedText()); - if (isMaterialized()) { - tbl.setRewriteEnabled(isRewriteEnabled()); - tbl.setTableType(TableType.MATERIALIZED_VIEW); - } else { - tbl.setTableType(TableType.VIRTUAL_VIEW); - } - tbl.setSerializationLib(null); - tbl.clearSerDeInfo(); - tbl.setFields(getSchema()); - if (getComment() != null) { - tbl.setProperty("comment", getComment()); - } - - if (tblProps != null) { - tbl.getParameters().putAll(tblProps); - } - - if (!CollectionUtils.isEmpty(partCols)) { - tbl.setPartCols(partCols); - } - - if (!CollectionUtils.isEmpty(sortColNames)) { - tbl.setProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS, - Utilities.encodeColumnNames(sortColNames)); - } - if (!CollectionUtils.isEmpty(distributeColNames)) { - tbl.setProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS, - Utilities.encodeColumnNames(distributeColNames)); - } - - if (getInputFormat() != null) { - tbl.setInputFormatClass(getInputFormat()); - } - - if (getOutputFormat() != null) { - tbl.setOutputFormatClass(getOutputFormat()); - } - - if (isMaterialized()) { - if (getLocation() != null) { - tbl.setDataLocation(new Path(getLocation())); - } - - if (getStorageHandler() != null) { - tbl.setProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, - getStorageHandler()); - } - HiveStorageHandler storageHandler = tbl.getStorageHandler(); - - /* - * If the user didn't specify a SerDe, we use the default. - */ - String serDeClassName; - if (getSerde() == null) { - if (storageHandler == null) { - serDeClassName = PlanUtils.getDefaultSerDe().getName(); - LOG.info("Default to {} for materialized view {}", serDeClassName, getViewName()); - } else { - serDeClassName = storageHandler.getSerDeClass().getName(); - LOG.info("Use StorageHandler-supplied {} for materialized view {}", serDeClassName, getViewName()); - } - } else { - // let's validate that the serde exists - serDeClassName = getSerde(); - DDLUtils.validateSerDe(serDeClassName, conf); - } - tbl.setSerializationLib(serDeClassName); - - // To remain consistent, we need to set input and output formats both - // at the table level and the storage handler level. - tbl.setInputFormatClass(getInputFormat()); - tbl.setOutputFormatClass(getOutputFormat()); - if (getInputFormat() != null && !getInputFormat().isEmpty()) { - tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName()); - } - if (getOutputFormat() != null && !getOutputFormat().isEmpty()) { - tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); - } - } - - if (ownerName != null) { - tbl.setOwner(ownerName); - } - - // Sets the column state for the create view statement (false since it is a creation). - // Similar to logic in CreateTableDesc. - StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null, - StatsSetupConst.FALSE); - - return tbl; - } - - public void setInitialMmWriteId(Long mmWriteId) { - this.initialMmWriteId = mmWriteId; - } - - public Long getInitialMmWriteId() { - return initialMmWriteId; - } - - public FileSinkDesc getAndUnsetWriter() { - FileSinkDesc fsd = writer; - writer = null; - return fsd; - } - - public void setWriter(FileSinkDesc writer) { - this.writer = writer; + return replicationSpec; } public void setOwnerName(String ownerName) { @@ -486,7 +112,6 @@ public void setOwnerName(String ownerName) { } public String getOwnerName() { - return this.ownerName; + return ownerName; } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index f7952a5cc1..4be0f5a151 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -18,10 +18,11 @@ package org.apache.hadoop.hive.ql.ddl.view.create; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; @@ -30,8 +31,8 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; - -import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.hive.ql.parse.HiveTableName; +import org.apache.hadoop.hive.ql.parse.StorageFormat; /** * Operation process of creating a view. @@ -45,19 +46,20 @@ public CreateViewOperation(DDLOperationContext context, CreateViewDesc desc) { public int execute() throws HiveException { Table oldview = context.getDb().getTable(desc.getViewName(), false); if (oldview != null) { + boolean isReplace = desc.isReplace(); + // Check whether we are replicating if (desc.getReplicationSpec().isInReplicationScope()) { // if this is a replication spec, then replace-mode semantics might apply. if (desc.getReplicationSpec().allowEventReplacementInto(oldview.getParameters())) { - desc.setReplace(true); // we replace existing view. + isReplace = true; // we replace existing view. } else { - LOG.debug("DDLTask: Create View is skipped as view {} is newer than update", - desc.getViewName()); // no replacement, the existing table state is newer than our update. + LOG.debug("DDLTask: Create View is skipped as view {} is newer than update", desc.getViewName()); return 0; } } - if (!desc.isReplace()) { + if (!isReplace) { if (desc.getIfNotExists()) { return 0; } @@ -66,27 +68,19 @@ public int execute() throws HiveException { throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName())); } - // It should not be a materialized view - assert !desc.isMaterialized(); - // replace existing view // remove the existing partition columns from the field schema - oldview.setViewOriginalText(desc.getViewOriginalText()); - oldview.setViewExpandedText(desc.getViewExpandedText()); + oldview.setViewOriginalText(desc.getOriginalText()); + oldview.setViewExpandedText(desc.getExpandedText()); oldview.setFields(desc.getSchema()); if (desc.getComment() != null) { oldview.setProperty("comment", desc.getComment()); } - if (desc.getTblProps() != null) { - oldview.getTTable().getParameters().putAll(desc.getTblProps()); - } - oldview.setPartCols(desc.getPartCols()); - if (desc.getInputFormat() != null) { - oldview.setInputFormatClass(desc.getInputFormat()); - } - if (desc.getOutputFormat() != null) { - oldview.setOutputFormatClass(desc.getOutputFormat()); + if (desc.getProperties() != null) { + oldview.getTTable().getParameters().putAll(desc.getProperties()); } + oldview.setPartCols(desc.getPartitionColumns()); + oldview.checkValidity(null); if (desc.getOwnerName() != null) { oldview.setOwner(desc.getOwnerName()); @@ -96,23 +90,54 @@ public int execute() throws HiveException { context.getWork().getOutputs()); } else { // We create new view - Table tbl = desc.toTable(context.getConf()); - // We set the signature for the view if it is a materialized view - if (tbl.isMaterializedView()) { - CreationMetadata cm = - new CreationMetadata(MetaStoreUtils.getDefaultCatalog(context.getConf()), tbl.getDbName(), - tbl.getTableName(), ImmutableSet.copyOf(desc.getTablesUsed())); - cm.setValidTxnList(context.getConf().get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); - tbl.getTTable().setCreationMetadata(cm); - } - context.getDb().createTable(tbl, desc.getIfNotExists()); - DDLUtils.addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK), + Table view = createViewObject(); + context.getDb().createTable(view, desc.getIfNotExists()); + DDLUtils.addIfAbsentByName(new WriteEntity(view, WriteEntity.WriteType.DDL_NO_LOCK), context.getWork().getOutputs()); //set lineage info - DataContainer dc = new DataContainer(tbl.getTTable()); - context.getQueryState().getLineageState().setLineage(new Path(desc.getViewName()), dc, tbl.getCols()); + DataContainer dc = new DataContainer(view.getTTable()); + context.getQueryState().getLineageState().setLineage(new Path(desc.getViewName()), dc, view.getCols()); } return 0; } + + private Table createViewObject() throws HiveException { + TableName name = HiveTableName.of(desc.getViewName()); + Table view = new Table(name.getDb(), name.getTable()); + view.setViewOriginalText(desc.getOriginalText()); + view.setViewExpandedText(desc.getExpandedText()); + view.setTableType(TableType.VIRTUAL_VIEW); + view.setSerializationLib(null); + view.clearSerDeInfo(); + view.setFields(desc.getSchema()); + if (desc.getComment() != null) { + view.setProperty("comment", desc.getComment()); + } + + if (desc.getProperties() != null) { + view.getParameters().putAll(desc.getProperties()); + } + + if (!CollectionUtils.isEmpty(desc.getPartitionColumns())) { + view.setPartCols(desc.getPartitionColumns()); + } + + StorageFormat storageFormat = new StorageFormat(context.getConf()); + storageFormat.fillDefaultStorageFormat(false, false); + + view.setInputFormatClass(storageFormat.getInputFormat()); + view.setOutputFormatClass(storageFormat.getOutputFormat()); + + if (desc.getOwnerName() != null) { + view.setOwner(desc.getOwnerName()); + } + + // Sets the column state for the create view statement (false since it is a creation). + // Similar to logic in CreateTableDesc. + StatsSetupConst.setStatsStateForCreateTable(view.getTTable().getParameters(), null, + StatsSetupConst.FALSE); + + return view; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index b578d48ce1..15ea0f6a56 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -303,20 +303,17 @@ a database ( directory ) String dbName = dbNameToLoadIn == null ? table.getDbName() : dbNameToLoadIn; TableName tableName = HiveTableName.ofNullable(table.getTableName(), dbName); String dbDotView = tableName.getNotEmptyDbTable(); - CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getAllCols(), null, table.getParameters(), - table.getPartColNames(), false, false, false, table.getSd().getInputFormat(), table.getSd().getOutputFormat(), - table.getSd().getSerdeInfo().getSerializationLib()); - String originalText = table.getViewOriginalText(); - String expandedText = table.getViewExpandedText(); + String viewOriginalText = table.getViewOriginalText(); + String viewExpandedText = table.getViewExpandedText(); if (!dbName.equals(table.getDbName())) { // TODO: If the DB name doesn't match with the metadata from dump, then need to rewrite the original and expanded // texts using new DB name. Currently it refers to the source database name. } - desc.setViewOriginalText(originalText); - desc.setViewExpandedText(expandedText); - desc.setPartCols(table.getPartCols()); + CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getAllCols(), null, table.getParameters(), + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 4f1e23d7a6..b30b9a0f66 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -191,7 +191,7 @@ public String getCboInfo() { return ctx.getCboInfo(); } - class RowFormatParams { + protected class RowFormatParams { String fieldDelim = null; String fieldEscape = null; String collItemDelim = null; @@ -199,7 +199,10 @@ public String getCboInfo() { String lineDelim = null; String nullFormat = null; - protected void analyzeRowFormat(ASTNode child) throws SemanticException { + public RowFormatParams() { + } + + public void analyzeRowFormat(ASTNode child) throws SemanticException { child = (ASTNode) child.getChild(0); int numChildRowFormat = child.getChildCount(); for (int numC = 0; numC < numChildRowFormat; numC++) { @@ -1815,7 +1818,7 @@ public void addPropertyReadEntry(Map tblProps, Set i /** * Unparses the analyzed statement */ - protected void executeUnparseTranlations() { + public void executeUnparseTranlations() { UnparseTranslator unparseTranslator = new UnparseTranslator(conf); unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index ee3aaa5799..cfaddbd5e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -546,6 +546,10 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept } // 2. Regen OP plan from optimized AST + if (forViewCreation) { + // the reset would remove the translations + executeUnparseTranlations(); + } if (cboCtx.type == PreCboCtx.Type.VIEW) { try { viewSelect = handleCreateViewDDL(newAST); @@ -1827,7 +1831,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu calciteGenPlan = genLogicalPlan(getQB(), true, null, null); // if it is to create view, we do not use table alias resultSchema = convertRowSchemaToResultSetSchema(relToHiveRR.get(calciteGenPlan), - getQB().isView() || getQB().isMaterializedView() ? false : HiveConf.getBoolVar(conf, + getQB().isMaterializedView() ? false : HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_RESULTSET_USE_UNIQUE_COLUMN_NAMES)); } catch (SemanticException e) { semanticException = e; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index bef02176c2..db73014108 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.ddl.view.materialized.update.MaterializedViewUpdateDesc; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -118,7 +118,7 @@ private AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc createTableDesc; - private CreateViewDesc createViewDesc; + private CreateMaterializedViewDesc createViewDesc; private MaterializedViewUpdateDesc materializedViewUpdateDesc; private boolean reduceSinkAddedBySortedDynPartition; @@ -193,7 +193,7 @@ public ParseContext( Map viewAliasToInput, List reduceSinkOperatorsAddedByEnforceBucketingSorting, AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc, - CreateViewDesc createViewDesc, MaterializedViewUpdateDesc materializedViewUpdateDesc, + CreateMaterializedViewDesc createViewDesc, MaterializedViewUpdateDesc materializedViewUpdateDesc, QueryProperties queryProperties, Map viewProjectToTableSchema) { this.queryState = queryState; @@ -591,7 +591,7 @@ public void setCreateTable(CreateTableDesc createTableDesc) { this.createTableDesc = createTableDesc; } - public CreateViewDesc getCreateViewDesc() { + public CreateMaterializedViewDesc getCreateViewDesc() { return createViewDesc; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 9d94b6e2dd..c5f2ef3132 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -29,9 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.metadata.Table; /** @@ -64,7 +62,7 @@ private Set aliasInsideView; // If this is a materialized view, this stores the view descriptor - private CreateViewDesc viewDesc; + private CreateMaterializedViewDesc viewDesc; // used by PTFs /* @@ -407,20 +405,16 @@ public QBSubQuery getHavingClauseSubQueryPredicate() { return havingClauseSubQueryPredicate; } - public CreateViewDesc getViewDesc() { + public CreateMaterializedViewDesc getViewDesc() { return viewDesc; } - public void setViewDesc(CreateViewDesc viewDesc) { + public void setViewDesc(CreateMaterializedViewDesc viewDesc) { this.viewDesc = viewDesc; } public boolean isMaterializedView() { - return viewDesc != null && viewDesc.isMaterialized(); - } - - public boolean isView() { - return viewDesc != null && !viewDesc.isMaterialized(); + return viewDesc != null; } public boolean isMultiDestQuery() { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e13dfe82b7..09dfdab228 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -112,7 +112,7 @@ import org.apache.hadoop.hive.ql.ddl.table.misc.preinsert.PreInsertTableDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.properties.AlterTableUnsetPropertiesDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.skewed.SkewedTableUtils; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.ddl.view.materialized.update.MaterializedViewUpdateDesc; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; import org.apache.hadoop.hive.ql.exec.ArchiveUtils; @@ -342,6 +342,7 @@ private Map opToSamplePruner; private final Map> opToPartToSkewedPruner; private Map viewProjectToTableSchema; + private Operator sinkOp; /** * a map for the split sampling, from alias to an instance of SplitSample * that describes percentage and number. @@ -350,7 +351,7 @@ private Map> groupOpToInputTables; protected Map prunedPartitions; protected List resultSchema; - protected CreateViewDesc createVwDesc; + protected CreateMaterializedViewDesc createVwDesc; private MaterializedViewUpdateDesc materializedViewUpdateDesc; private List viewsExpanded; protected ASTNode viewSelect; @@ -374,6 +375,9 @@ // flag for no scan during analyze ... compute statistics private boolean noscan; + // flag indicating that the analyzations should go only till resultSchema is ready + protected boolean forViewCreation; + // whether this is a mv rebuild rewritten expression protected MaterializationRebuildMode mvRebuildMode = MaterializationRebuildMode.NONE; protected String mvRebuildDbName; // Db name for materialization to rebuild @@ -7658,7 +7662,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) TableName tableName = null; Map tblProps = null; CreateTableDesc tblDesc = qb.getTableDesc(); - CreateViewDesc viewDesc = qb.getViewDesc(); + CreateMaterializedViewDesc viewDesc = qb.getViewDesc(); if (tblDesc != null) { fieldSchemas = new ArrayList<>(); partitionColumns = new ArrayList<>(); @@ -12125,7 +12129,7 @@ protected Table getTableObjectByName(String tableName, boolean throwException) t } } - private Table getTableObjectByName(String tableName) throws HiveException { + public Table getTableObjectByName(String tableName) throws HiveException { return getTableObjectByName(tableName, true); } @@ -12408,10 +12412,7 @@ boolean genResolvedParseTree(ASTNode ast, PlannerContext plannerCtx) throws Sema } // 3. analyze create view command - if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || - ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW || - (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && - ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { + if (ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW) { child = analyzeCreateView(ast, qb, plannerCtx); if (child == null) { return false; @@ -12590,7 +12591,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE ASTNode astForMasking; if (isCBOExecuted() && needsTransform && - (qb.isCTAS() || qb.isView() || qb.isMaterializedView() || qb.isMultiDestQuery())) { + (qb.isCTAS() || qb.isMaterializedView() || qb.isMultiDestQuery())) { // If we use CBO and we may apply masking/filtering policies, we create a copy of the ast. // The reason is that the generation of the operator tree may modify the initial ast, // but if we need to parse for a second time, we would like to parse the unmodified ast. @@ -12600,7 +12601,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE } // 2. Gen OP Tree from resolved Parse Tree - Operator sinkOp = genOPTree(ast, plannerCtx); + sinkOp = genOPTree(ast, plannerCtx); boolean usesMasking = false; if (!unparseTranslator.isEnabled() && @@ -12666,6 +12667,25 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE // Set the mapjoin hint if it needs to be disabled. pCtx.setDisableMapJoin(disableMapJoinWithHint(getQB().getParseInfo().getHintList())); + if (forViewCreation) { + // Generate lineage info if LineageLogger hook is configured. + // Add the transformation that computes the lineage information. + Set postExecHooks = Sets.newHashSet(Splitter.on(",").trimResults() + .omitEmptyStrings() + .split(Strings.nullToEmpty(HiveConf.getVar(conf, HiveConf.ConfVars.POSTEXECHOOKS)))); + if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter") + || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger") + || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) { + List transformations = new ArrayList(); + transformations.add(new HiveOpConverterPostProc()); + transformations.add(new Generator(postExecHooks)); + for (Transform t : transformations) { + pCtx = t.transform(pCtx); + } + } + return; + } + // 5. Take care of view creation if (createVwDesc != null) { if (ctx.getExplainAnalyze() == AnalyzeState.RUNNING) { @@ -12680,42 +12700,7 @@ void analyzeInternal(ASTNode ast, Supplier pcf) throws SemanticE // all the information for semanticcheck validateCreateView(); - if (createVwDesc.isMaterialized()) { - createVwDesc.setTablesUsed(getTablesUsed(pCtx)); - } else { - // Since we're only creating a view (not executing it), we don't need to - // optimize or translate the plan (and in fact, those procedures can - // interfere with the view creation). So skip the rest of this method. - ctx.setResDir(null); - ctx.setResFile(null); - - try { - PlanUtils.addInputsForView(pCtx); - } catch (HiveException e) { - throw new SemanticException(e); - } - - // Generate lineage info for create view statements - // if LineageLogger hook is configured. - // Add the transformation that computes the lineage information. - Set postExecHooks = Sets.newHashSet(Splitter.on(",").trimResults() - .omitEmptyStrings() - .split(Strings.nullToEmpty(HiveConf.getVar(conf, HiveConf.ConfVars.POSTEXECHOOKS)))); - if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter") - || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger") - || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) { - List transformations = new ArrayList(); - transformations.add(new HiveOpConverterPostProc()); - transformations.add(new Generator(postExecHooks)); - for (Transform t : transformations) { - pCtx = t.transform(pCtx); - } - // we just use view name as location. - queryState.getLineageState() - .mapDirToOp(new Path(createVwDesc.getViewName()), sinkOp); - } - return; - } + createVwDesc.setTablesUsed(getTablesUsed(pCtx)); } // 6. Generate table access stats if required @@ -12835,11 +12820,6 @@ private void putAccessedColumnsToReadEntity(Set inputs, ColumnAccess } protected void saveViewDefinition() throws SemanticException { - if (createVwDesc.isMaterialized() && createVwDesc.isReplace()) { - // This is a rebuild, there's nothing to do here - return; - } - // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List derivedSchema = @@ -12871,116 +12851,34 @@ protected void saveViewDefinition() throws SemanticException { String expandedText = ctx.getTokenRewriteStream().toString( viewSelect.getTokenStartIndex(), viewSelect.getTokenStopIndex()); - if (createVwDesc.isMaterialized()) { - if (createVwDesc.getPartColNames() != null) { - // If we are creating a materialized view and it has partition columns, - // we may need to reorder column projection in expanded query. The reason - // is that Hive assumes that in the partition columns are at the end of - // the MV schema, and if we do not do this, we will have a mismatch between - // the SQL query for the MV and the MV itself. - boolean first = true; - StringBuilder sb = new StringBuilder(); - sb.append("SELECT "); - for (FieldSchema fieldSchema : derivedSchema) { - if (!createVwDesc.getPartColNames().contains(fieldSchema.getName())) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)); - } - } - for (String partColName : createVwDesc.getPartColNames()) { - sb.append(", "); - sb.append(HiveUtils.unparseIdentifier(partColName, conf)); - } - sb.append(" FROM ("); - sb.append(expandedText); - sb.append(") "); - sb.append(HiveUtils.unparseIdentifier(Utilities.getDbTableName(createVwDesc.getViewName())[1], conf)); - expandedText = sb.toString(); - } - } else { - if (imposedSchema != null) { - // Merge the names from the imposed schema into the types - // from the derived schema. - StringBuilder sb = new StringBuilder(); - sb.append("SELECT "); - int n = derivedSchema.size(); - for (int i = 0; i < n; ++i) { - if (i > 0) { + if (createVwDesc.getPartColNames() != null) { + // If we are creating a materialized view and it has partition columns, + // we may need to reorder column projection in expanded query. The reason + // is that Hive assumes that in the partition columns are at the end of + // the MV schema, and if we do not do this, we will have a mismatch between + // the SQL query for the MV and the MV itself. + boolean first = true; + StringBuilder sb = new StringBuilder(); + sb.append("SELECT "); + for (FieldSchema fieldSchema : derivedSchema) { + if (!createVwDesc.getPartColNames().contains(fieldSchema.getName())) { + if (first) { + first = false; + } else { sb.append(", "); } - FieldSchema fieldSchema = derivedSchema.get(i); - // Modify a copy, not the original - fieldSchema = new FieldSchema(fieldSchema); - // TODO: there's a potential problem here if some table uses external schema like Avro, - // with a very large type name. It seems like the view does not derive the SerDe from - // the table, so it won't be able to just get the type from the deserializer like the - // table does; we won't be able to properly store the type in the RDBMS metastore. - // Not sure if these large cols could be in resultSchema. Ignore this for now 0_o - derivedSchema.set(i, fieldSchema); sb.append(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)); - sb.append(" AS "); - String imposedName = imposedSchema.get(i).getName(); - sb.append(HiveUtils.unparseIdentifier(imposedName, conf)); - fieldSchema.setName(imposedName); - // We don't currently allow imposition of a type - fieldSchema.setComment(imposedSchema.get(i).getComment()); - } - sb.append(" FROM ("); - sb.append(expandedText); - sb.append(") "); - sb.append(HiveUtils.unparseIdentifier(Utilities.getDbTableName(createVwDesc.getViewName())[1], conf)); - expandedText = sb.toString(); - } - - if (createVwDesc.getPartColNames() != null) { - // Make sure all partitioning columns referenced actually - // exist and are in the correct order at the end - // of the list of columns produced by the view. Also move the field - // schema descriptors from derivedSchema to the partitioning key - // descriptor. - List partColNames = createVwDesc.getPartColNames(); - if (partColNames.size() > derivedSchema.size()) { - throw new SemanticException( - ErrorMsg.VIEW_PARTITION_MISMATCH.getMsg()); - } - - // Get the partition columns from the end of derivedSchema. - List partitionColumns = derivedSchema.subList( - derivedSchema.size() - partColNames.size(), - derivedSchema.size()); - - // Verify that the names match the PARTITIONED ON clause. - Iterator colNameIter = partColNames.iterator(); - Iterator schemaIter = partitionColumns.iterator(); - while (colNameIter.hasNext()) { - String colName = colNameIter.next(); - FieldSchema fieldSchema = schemaIter.next(); - if (!fieldSchema.getName().equals(colName)) { - throw new SemanticException( - ErrorMsg.VIEW_PARTITION_MISMATCH.getMsg()); - } } - - // Boundary case: require at least one non-partitioned column - // for consistency with tables. - if (partColNames.size() == derivedSchema.size()) { - throw new SemanticException( - ErrorMsg.VIEW_PARTITION_TOTAL.getMsg()); - } - - // Now make a copy. - createVwDesc.setPartCols( - new ArrayList(partitionColumns)); - - // Finally, remove the partition columns from the end of derivedSchema. - // (Clearing the subList writes through to the underlying - // derivedSchema ArrayList.) - partitionColumns.clear(); } + for (String partColName : createVwDesc.getPartColNames()) { + sb.append(", "); + sb.append(HiveUtils.unparseIdentifier(partColName, conf)); + } + sb.append(" FROM ("); + sb.append(expandedText); + sb.append(") "); + sb.append(HiveUtils.unparseIdentifier(Utilities.getDbTableName(createVwDesc.getViewName())[1], conf)); + expandedText = sb.toString(); } // Set schema and expanded text for the view @@ -13872,16 +13770,12 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt List cols = null; boolean ifNotExists = false; boolean rewriteEnabled = true; - boolean orReplace = false; - boolean isAlterViewAs = false; String comment = null; ASTNode selectStmt = null; Map tblProps = null; List partColNames = null; List sortColNames = null; List distributeColNames = null; - boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW; - boolean isRebuild = false; String location = null; RowFormatParams rowFormatParams = new RowFormatParams(); StorageFormat storageFormat = new StorageFormat(conf); @@ -13901,9 +13795,6 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt case HiveParser.TOK_REWRITE_DISABLED: rewriteEnabled = false; break; - case HiveParser.TOK_ORREPLACE: - orReplace = true; - break; case HiveParser.TOK_QUERY: // For CBO if (plannerCtx != null) { @@ -13957,82 +13848,59 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt } } - storageFormat.fillDefaultStorageFormat(false, isMaterialized); - - if (ifNotExists && orReplace) { - throw new SemanticException("Can't combine IF NOT EXISTS and OR REPLACE."); - } + storageFormat.fillDefaultStorageFormat(false, true); - if (isMaterialized) { - if (!ifNotExists) { - // Verify that the table does not already exist - // dumpTable is only used to check the conflict for non-temporary tables - try { - Table dumpTable = db.newTable(dbDotTable); - if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.isExplainSkipExecution()) { - throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTable)); - } - } catch (HiveException e) { - throw new SemanticException(e); + if (!ifNotExists) { + // Verify that the table does not already exist + // dumpTable is only used to check the conflict for non-temporary tables + try { + Table dumpTable = db.newTable(dbDotTable); + if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.isExplainSkipExecution()) { + throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTable)); } + } catch (HiveException e) { + throw new SemanticException(e); } - if (partColNames != null && (distributeColNames != null || sortColNames != null)) { - // Verify that partition columns and data organization columns are not overlapping - Set partColNamesSet = new HashSet<>(partColNames); - if (distributeColNames != null) { - for (String colName : distributeColNames) { - if (partColNamesSet.contains(colName)) { - throw new SemanticException("Same column cannot be present in partition and cluster/distribute clause. " - + "Column name: " + colName); - } + } + if (partColNames != null && (distributeColNames != null || sortColNames != null)) { + // Verify that partition columns and data organization columns are not overlapping + Set partColNamesSet = new HashSet<>(partColNames); + if (distributeColNames != null) { + for (String colName : distributeColNames) { + if (partColNamesSet.contains(colName)) { + throw new SemanticException("Same column cannot be present in partition and cluster/distribute clause. " + + "Column name: " + colName); } } - if (sortColNames != null) { - for (String colName : sortColNames) { - if (partColNamesSet.contains(colName)) { - throw new SemanticException("Same column cannot be present in partition and cluster/sort clause. " - + "Column name: " + colName); - } + } + if (sortColNames != null) { + for (String colName : sortColNames) { + if (partColNamesSet.contains(colName)) { + throw new SemanticException("Same column cannot be present in partition and cluster/sort clause. " + + "Column name: " + colName); } } } } - if (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && - ast.getChild(1).getType() == HiveParser.TOK_QUERY) { - isAlterViewAs = true; - orReplace = true; - } - unparseTranslator.enable(); - if (isMaterialized) { - if (makeAcid()) { - if (tblProps == null) { - tblProps = new HashMap<>(); - } - tblProps = convertToAcidByDefault(storageFormat, dbDotTable, null, tblProps); + if (makeAcid()) { + if (tblProps == null) { + tblProps = new HashMap<>(); } - - createVwDesc = new CreateViewDesc( - dbDotTable, cols, comment, tblProps, partColNames, sortColNames, distributeColNames, - ifNotExists, isRebuild, rewriteEnabled, isAlterViewAs, - storageFormat.getInputFormat(), storageFormat.getOutputFormat(), - location, storageFormat.getSerde(), storageFormat.getStorageHandler(), - storageFormat.getSerdeProps()); - addDbAndTabToOutputs(new String[] {qualTabName.getDb(), qualTabName.getTable()}, TableType.MATERIALIZED_VIEW, - false, tblProps); - queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); - } else { - createVwDesc = new CreateViewDesc( - dbDotTable, cols, comment, tblProps, partColNames, - ifNotExists, orReplace, isAlterViewAs, storageFormat.getInputFormat(), - storageFormat.getOutputFormat(), storageFormat.getSerde()); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), createVwDesc))); - addDbAndTabToOutputs(new String[] {qualTabName.getDb(), qualTabName.getTable()}, - TableType.VIRTUAL_VIEW, false, tblProps); - queryState.setCommandType(HiveOperation.CREATEVIEW); + tblProps = convertToAcidByDefault(storageFormat, dbDotTable, null, tblProps); } + + createVwDesc = new CreateMaterializedViewDesc( + dbDotTable, cols, comment, tblProps, partColNames, sortColNames, distributeColNames, + ifNotExists, rewriteEnabled, + storageFormat.getInputFormat(), storageFormat.getOutputFormat(), + location, storageFormat.getSerde(), storageFormat.getStorageHandler(), + storageFormat.getSerdeProps()); + addDbAndTabToOutputs(new String[] {qualTabName.getDb(), qualTabName.getTable()}, TableType.MATERIALIZED_VIEW, + false, tblProps); + queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); qb.setViewDesc(createVwDesc); return selectStmt; @@ -14065,8 +13933,7 @@ private void validateCreateView() if (table.isMaterializedView()) { throw new SemanticException("View definition references materialized view " + alias); } - if (createVwDesc.isMaterialized() && createVwDesc.isRewriteEnabled() && - !AcidUtils.isTransactionalTable(table)) { + if (createVwDesc.isRewriteEnabled() && !AcidUtils.isTransactionalTable(table)) { throw new SemanticException("Automatic rewriting for materialized view cannot " + "be enabled if the materialized view uses non-transactional tables"); } @@ -14075,11 +13942,11 @@ private void validateCreateView() } } - if (createVwDesc.isMaterialized() && !qb.hasTableDefined()) { + if (!qb.hasTableDefined()) { throw new SemanticException("Materialized view must have a table defined."); } - if (createVwDesc.isMaterialized() && createVwDesc.isRewriteEnabled()) { + if (createVwDesc.isRewriteEnabled()) { if (!ctx.isCboSucceeded()) { String msg = "Cannot enable automatic rewriting for materialized view."; if (ctx.getCboInfo() != null) { @@ -14092,60 +13959,6 @@ private void validateCreateView() getInvalidAutomaticRewritingMaterializationReason()); } } - - // ALTER VIEW AS SELECT requires the view must exist - if (createVwDesc.getIsAlterViewAs() && oldView == null) { - String viewNotExistErrorMsg = - "The following view does not exist: " + createVwDesc.getViewName(); - throw new SemanticException( - ErrorMsg.ALTER_VIEW_AS_SELECT_NOT_EXIST.getMsg(viewNotExistErrorMsg)); - } - - //replace view - if (createVwDesc.isReplace() && oldView != null) { - - // Don't allow swapping between virtual and materialized view in replace - if (oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && createVwDesc.isMaterialized()) { - throw new SemanticException(ErrorMsg.REPLACE_VIEW_WITH_MATERIALIZED, - oldView.getTableName()); - } - - if (oldView.getTableType().equals(TableType.MATERIALIZED_VIEW) && - !createVwDesc.isMaterialized()) { - throw new SemanticException(ErrorMsg.REPLACE_MATERIALIZED_WITH_VIEW, - oldView.getTableName()); - } - - // Existing table is not a view - if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && - !oldView.getTableType().equals(TableType.MATERIALIZED_VIEW)) { - String tableNotViewErrorMsg = - "The following is an existing table, not a view: " + - createVwDesc.getViewName(); - throw new SemanticException( - ErrorMsg.EXISTING_TABLE_IS_NOT_VIEW.getMsg(tableNotViewErrorMsg)); - } - - if (!createVwDesc.isMaterialized()) { - // if old view has partitions, it could not be replaced - String partitionViewErrorMsg = - "The following view has partition, it could not be replaced: " + - createVwDesc.getViewName(); - try { - if ((createVwDesc.getPartCols() == null || - createVwDesc.getPartCols().isEmpty() || - !createVwDesc.getPartCols().equals(oldView.getPartCols())) && - !oldView.getPartCols().isEmpty() && - !db.getPartitions(oldView).isEmpty()) { - throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); - } - } catch (HiveException e) { - throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); - } - } - } } catch (HiveException e) { throw new SemanticException(e.getMessage(), e); } @@ -15436,6 +15249,22 @@ public boolean isValidQueryCaching() { return (invalidResultCacheReason == null); } + public void setForViewCreation(boolean forViewCreation) { + this.forViewCreation = forViewCreation; + } + + public Map getTopOps() { + return topOps; + } + + public Map getViewAliasToInput() { + return viewAliasToInput; + } + + public Operator getSinkOp() { + return sinkOp; + } + protected enum MaterializationRebuildMode { NONE, INSERT_OVERWRITE_REBUILD, @@ -15509,7 +15338,7 @@ public WriteEntity getAcidAnalyzeTable() { } @Override - protected void executeUnparseTranlations() { + public void executeUnparseTranlations() { unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 2350646c36..b9f283f25b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -104,7 +104,7 @@ protected void processStorageFormat(String name) throws SemanticException { } } - protected void fillDefaultStorageFormat(boolean isExternal, boolean isMaterializedView) + public void fillDefaultStorageFormat(boolean isExternal, boolean isMaterializedView) throws SemanticException { if ((inputFormat == null) && (storageHandler == null)) { String defaultFormat; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 2f3fc6c50a..0ef11b2f1f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.ddl.view.materialized.alter.rewrite.AlterMaterializedViewRewriteDesc; import org.apache.hadoop.hive.ql.ddl.view.materialized.update.MaterializedViewUpdateDesc; import org.apache.hadoop.hive.ql.exec.FetchTask; @@ -366,7 +366,7 @@ public void compile(final ParseContext pCtx, CollectionUtils.isEmpty(crtTblDesc.getPartColNames())); } else if (pCtx.getQueryProperties().isMaterializedView()) { // generate a DDL task and make it a dependent task of the leaf - CreateViewDesc viewDesc = pCtx.getCreateViewDesc(); + CreateMaterializedViewDesc viewDesc = pCtx.getCreateViewDesc(); Task crtViewTask = TaskFactory.get(new DDLWork( inputs, outputs, viewDesc)); patchUpAfterCTASorMaterializedView(rootTasks, inputs, outputs, crtViewTask, @@ -454,7 +454,7 @@ private void setLoadFileLocation( txnId = ctd.getInitialMmWriteId(); loc = ctd.getLocation(); } else { - CreateViewDesc cmv = pCtx.getCreateViewDesc(); + CreateMaterializedViewDesc cmv = pCtx.getCreateViewDesc(); dataSink = cmv.getAndUnsetWriter(); txnId = cmv.getInitialMmWriteId(); loc = cmv.getLocation(); @@ -555,16 +555,14 @@ private void patchUpAfterCTASorMaterializedView(List> rootTasks, DDLTask ddlTask = (DDLTask)createTask; DDLWork work = ddlTask.getWork(); DDLDesc desc = work.getDDLDesc(); - if (desc instanceof CreateViewDesc) { - CreateViewDesc createViewDesc = (CreateViewDesc)desc; - if (createViewDesc.isMaterialized()) { - String tableName = createViewDesc.getViewName(); - boolean retrieveAndInclude = createViewDesc.isRewriteEnabled(); - MaterializedViewUpdateDesc materializedViewUpdateDesc = - new MaterializedViewUpdateDesc(tableName, retrieveAndInclude, false, false); - DDLWork ddlWork = new DDLWork(inputs, outputs, materializedViewUpdateDesc); - targetTask.addDependentTask(TaskFactory.get(ddlWork, conf)); - } + if (desc instanceof CreateMaterializedViewDesc) { + CreateMaterializedViewDesc createViewDesc = (CreateMaterializedViewDesc)desc; + String tableName = createViewDesc.getViewName(); + boolean retrieveAndInclude = createViewDesc.isRewriteEnabled(); + MaterializedViewUpdateDesc materializedViewUpdateDesc = + new MaterializedViewUpdateDesc(tableName, retrieveAndInclude, false, false); + DDLWork ddlWork = new DDLWork(inputs, outputs, materializedViewUpdateDesc); + targetTask.addDependentTask(TaskFactory.get(ddlWork, conf)); } else if (desc instanceof AlterMaterializedViewRewriteDesc) { AlterMaterializedViewRewriteDesc alterMVRewriteDesc = (AlterMaterializedViewRewriteDesc)desc; String tableName = alterMVRewriteDesc.getMaterializedViewName(); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index c75829c272..c72ebb3014 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -109,8 +109,8 @@ RELOADFUNCTION("RELOADFUNCTION", HiveParser.TOK_RELOADFUNCTIONS, null, null), CREATEMACRO("CREATEMACRO", HiveParser.TOK_CREATEMACRO, null, null), DROPMACRO("DROPMACRO", HiveParser.TOK_DROPMACRO, null, null), - CREATEVIEW("CREATEVIEW", HiveParser.TOK_CREATEVIEW, new Privilege[]{Privilege.SELECT}, - new Privilege[]{Privilege.CREATE}), + CREATEVIEW("CREATEVIEW", new int[] {HiveParser.TOK_CREATEVIEW, HiveParser.TOK_ALTERVIEW_AS}, + new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE}), CREATE_MATERIALIZED_VIEW("CREATE_MATERIALIZED_VIEW", HiveParser.TOK_CREATE_MATERIALIZED_VIEW, new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE}), DROPVIEW("DROPVIEW", HiveParser.TOK_DROPVIEW, null, new Privilege[]{Privilege.DROP}), diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java index 07bcef8ee3..d9246b3520 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java @@ -22,7 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -38,7 +38,7 @@ private String columns; private String columnTypes; private transient CreateTableDesc ctasCreateTableDesc; - private transient CreateViewDesc createViewDesc; + private transient CreateMaterializedViewDesc createViewDesc; private boolean isMmCtas; public LoadFileDesc(final LoadFileDesc o) { @@ -53,14 +53,14 @@ public LoadFileDesc(final LoadFileDesc o) { this.createViewDesc = o.createViewDesc; } - public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, + public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateMaterializedViewDesc createViewDesc, final Path sourcePath, final Path targetDir, final boolean isDfsDir, final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas); if (createTableDesc != null && createTableDesc.isCTAS()) { this.ctasCreateTableDesc = createTableDesc; } - if (createViewDesc != null && createViewDesc.isMaterialized()) { + if (createViewDesc != null) { this.createViewDesc = createViewDesc; } } @@ -136,7 +136,7 @@ public CreateTableDesc getCtasCreateTableDesc() { return ctasCreateTableDesc; } - public CreateViewDesc getCreateViewDesc() { + public CreateMaterializedViewDesc getCreateViewDesc() { return createViewDesc; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 980f39b681..7cf7376084 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateMaterializedViewDesc; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -409,7 +409,7 @@ public static TableDesc getTableDesc(CreateTableDesc crtTblDesc, String cols, /** * Generate a table descriptor from a createViewDesc. */ - public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, String colTypes) { + public static TableDesc getTableDesc(CreateMaterializedViewDesc crtViewDesc, String cols, String colTypes) { TableDesc ret; try { @@ -443,7 +443,7 @@ public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, St crtViewDesc.getStorageHandler()); } - if (crtViewDesc.getViewName() != null && crtViewDesc.isMaterialized()) { + if (crtViewDesc.getViewName() != null) { properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, crtViewDesc.getViewName()); }