commit 731877a4e5080bf1431f0777b24904b4f782b4b3 Author: Alan Gates Date: Tue May 12 15:22:55 2015 -0700 HIVE-10459 Initial materialized view patch HIVE-10459 Removed some dead code, updated create test. HIVE-10459 Removed partitions from materialized view definition since we don't support it anyway. HIVE-10459 Changed drop test to actually test drop. HIVE-10495 Added negative test for illegitamite replace. HIVE-10459 WIP, working on security changes HIVE-10459 Added a bunch of qfile tests. HIVE-10459 Added checked to prevent update/delete and many qfile tests diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index eff4d30..ab0063f 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -836,6 +836,11 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "Default file format for CREATE TABLE statement applied to managed tables only. External tables will be \n" + "created with format specified by hive.default.fileformat. Leaving this null will result in using hive.default.fileformat \n" + "for all tables."), + HIVEMATERIALIZEDVIEWFILEFORMAT("hive.materializedview.fileformat", "ORC", + new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), + "File format for CREATE MATERIALIZED VIEW statement."), + HIVEMATERIALIZEDVIEWSERDE("hive.materializedview.serde", + "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "Serde used for materialized views"), HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile", new StringSet("TextFile", "SequenceFile", "RCfile"), "Default file format for storing result of the query."), HIVECHECKFILEFORMAT("hive.fileformat.check", true, "Whether to check file format or not when loading data files"), diff --git jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java index 13e42b5..a94c926 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java @@ -694,6 +694,8 @@ public static String toJdbcTableType(String hivetabletype) { return "VIEW"; } else if (hivetabletype.equals(TableType.EXTERNAL_TABLE.toString())) { return "EXTERNAL TABLE"; + } else if (hivetabletype.equals(TableType.MATERIALIZED_VIEW.toString())) { + return "MATERIALIZED VIEW"; } else { return hivetabletype; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java index 56eeaa0..e9e16d7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java @@ -22,5 +22,5 @@ * Typesafe enum for types of tables described by the metastore. */ public enum TableType { - MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE + MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE, MATERIALIZED_VIEW } diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index e1cbaa6..fdafe83 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -432,6 +432,11 @@ UPDATE_CANNOT_UPDATE_BUCKET_VALUE(10302, "Updating values of bucketing columns is not supported. Column {0}.", true), IMPORT_INTO_STRICT_REPL_TABLE(10303,"Non-repl import disallowed against table that is a destination of replication."), + REPLACE_VIEW_WITH_MATERIALIZED(10302, "Attempt to replace view {0} with materialized view", true), + REPLACE_MATERIALIZED_WITH_VIEW(10303, "Attempt to replace materialized view {0} with view", true), + + UPDATE_DELETE_VIEW(10304, "You cannot update or delete records in a view"), + //========================== 20000 range starts here ========================// SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."), SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. " diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index e8f7fba..2204a86 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -66,6 +66,8 @@ private boolean multiDestQuery; private boolean filterWithSubQuery; + // True if this statement creates or replaces a materialized view + private boolean isMaterializedView; public boolean isQuery() { return query; @@ -269,6 +271,19 @@ public boolean hasFilterWithSubQuery() { return this.filterWithSubQuery; } + /** + * True indicates this statement create or replaces a materialized view, not that it is a query + * against a materialized view. + * @return + */ + public boolean isMaterializedView() { + return isMaterializedView; + } + + public void setMaterializedView(boolean isMaterializedView) { + this.isMaterializedView = isMaterializedView; + } + public void clear() { query = false; analyzeCommand = false; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 04c89ae..65ea00a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4153,7 +4153,8 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws Exceptio // Get the existing table Table oldtbl = db.getTable(crtTbl.getLikeTableName()); Table tbl; - if (oldtbl.getTableType() == TableType.VIRTUAL_VIEW) { + if (oldtbl.getTableType() == TableType.VIRTUAL_VIEW || + oldtbl.getTableType() == TableType.MATERIALIZED_VIEW) { String targetTableName = crtTbl.getTableName(); tbl=db.newTable(targetTableName); @@ -4291,33 +4292,44 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws Exceptio private int createView(Hive db, CreateViewDesc crtView) throws HiveException { Table oldview = db.getTable(crtView.getViewName(), false); if (crtView.getOrReplace() && oldview != null) { - // replace existing view - // remove the existing partition columns from the field schema - oldview.setViewOriginalText(crtView.getViewOriginalText()); - oldview.setViewExpandedText(crtView.getViewExpandedText()); - oldview.setFields(crtView.getSchema()); - if (crtView.getComment() != null) { - oldview.setProperty("comment", crtView.getComment()); - } - if (crtView.getTblProps() != null) { - oldview.getTTable().getParameters().putAll(crtView.getTblProps()); - } - oldview.setPartCols(crtView.getPartCols()); - oldview.checkValidity(); - try { - db.alterTable(crtView.getViewName(), oldview); - } catch (InvalidOperationException e) { - throw new HiveException(e); + if (!crtView.isMaterialized()) { + // replace existing view + // remove the existing partition columns from the field schema + oldview.setViewOriginalText(crtView.getViewOriginalText()); + oldview.setViewExpandedText(crtView.getViewExpandedText()); + oldview.setFields(crtView.getSchema()); + if (crtView.getComment() != null) { + oldview.setProperty("comment", crtView.getComment()); + } + if (crtView.getTblProps() != null) { + oldview.getTTable().getParameters().putAll(crtView.getTblProps()); + } + oldview.setPartCols(crtView.getPartCols()); + oldview.checkValidity(); + try { + db.alterTable(crtView.getViewName(), oldview); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } + work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); + } else { + // This is a replace, so we need an exclusive lock + work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE)); } - work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); } else { // create new view Table tbl = db.newTable(crtView.getViewName()); - tbl.setTableType(TableType.VIRTUAL_VIEW); + if (crtView.isMaterialized()) { + tbl.setTableType(TableType.MATERIALIZED_VIEW); + } else { + tbl.setTableType(TableType.VIRTUAL_VIEW); + } tbl.setSerializationLib(null); tbl.clearSerDeInfo(); tbl.setViewOriginalText(crtView.getViewOriginalText()); - tbl.setViewExpandedText(crtView.getViewExpandedText()); + if (!crtView.isMaterialized()) { + tbl.setViewExpandedText(crtView.getViewExpandedText()); + } tbl.setFields(crtView.getSchema()); if (crtView.getComment() != null) { tbl.setProperty("comment", crtView.getComment()); @@ -4330,6 +4342,15 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { tbl.setPartCols(crtView.getPartCols()); } + if (crtView.isMaterialized()) { + // Short circuit the checks that the input format is valid, this is configured for all + // materialized views and doesn't change so we don't need to check it constantly. + tbl.getSd().setInputFormat(crtView.getInputFormat()); + tbl.getSd().setOutputFormat(crtView.getOutputFormat()); + tbl.getSd().setSerdeInfo(new SerDeInfo(crtView.getSerde(), crtView.getSerde(), + Collections.emptyMap())); + } + db.createTable(tbl, crtView.getIfNotExists()); work.getOutputs().add(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index e53933e..0847a3f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -206,8 +206,11 @@ public void checkValidity() throws HiveException { } if (isView()) { + assert (getViewOriginalText() != null); + assert (getViewExpandedText() != null); + } else if (isMaterializedView()) { assert(getViewOriginalText() != null); - assert(getViewExpandedText() != null); + assert(getViewExpandedText() == null); } else { assert(getViewOriginalText() == null); assert(getViewExpandedText() == null); @@ -798,6 +801,10 @@ public boolean isView() { return TableType.VIRTUAL_VIEW.equals(getTableType()); } + public boolean isMaterializedView() { + return TableType.MATERIALIZED_VIEW.equals(getTableType()); + } + /** * @return whether this table is actually an index table */ diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index d72991f..0495641 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -750,7 +750,9 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit assert (ast.getToken().getType() == HiveParser.TOK_TAB || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION || ast.getToken().getType() == HiveParser.TOK_TABTYPE - || ast.getToken().getType() == HiveParser.TOK_CREATETABLE); + || ast.getToken().getType() == HiveParser.TOK_CREATETABLE + || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW + || ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW); int childIndex = 0; numDynParts = 0; @@ -762,7 +764,9 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit tableName = conf.getVar(HiveConf.ConfVars.HIVETESTMODEPREFIX) + tableName; } - if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE && + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_REBUILD_MATERIALIZED_VIEW) { tableHandle = db.getTable(tableName); } } catch (InvalidTableException ite) { @@ -774,7 +778,9 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit } // get partition metadata if partition specified - if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE && + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_REBUILD_MATERIALIZED_VIEW) { childIndex = 1; ASTNode partspec = (ASTNode) ast.getChild(1); partitions = new ArrayList(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index bc69d23..39a9eb7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1112,7 +1112,7 @@ private void analyzeCreateIndex(ASTNode ast) throws SemanticException { } } - storageFormat.fillDefaultStorageFormat(false); + storageFormat.fillDefaultStorageFormat(false, false); if (indexTableName == null) { indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName); indexTableName = qTabName[0] + "." + indexTableName; // on same database with base table diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index a4c5d0e..53f4d30 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -134,7 +134,7 @@ static void validateTable(org.apache.hadoop.hive.ql.metadata.Table table) throws ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(":Table " + table.getTableName())); } - if (table.isView()) { + if (table.isView() || table.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } if (table.isNonNative()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 4bb256d..d944913 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -239,6 +239,8 @@ TOK_ALTERVIEW_DROPPROPERTIES; TOK_ALTERVIEW_ADDPARTS; TOK_ALTERVIEW_DROPPARTS; TOK_ALTERVIEW_RENAME; +TOK_REBUILD_MATERIALIZED_VIEW; +TOK_CREATE_MATERIALIZED_VIEW; TOK_VIEWPARTCOLS; TOK_EXPLAIN; TOK_EXPLAIN_SQ_REWRITE; @@ -742,7 +744,9 @@ ddlStatement | showStatement | metastoreCheck | createViewStatement + | createMaterializedViewStatement | dropViewStatement + | dropMaterializedViewStatement | createFunctionStatement | createMacroStatement | createIndexStatement @@ -994,6 +998,7 @@ alterStatement @after { popMsg(state); } : KW_ALTER KW_TABLE tableName alterTableStatementSuffix -> ^(TOK_ALTERTABLE tableName alterTableStatementSuffix) | KW_ALTER KW_VIEW tableName KW_AS? alterViewStatementSuffix -> ^(TOK_ALTERVIEW tableName alterViewStatementSuffix) + | KW_ALTER KW_MATERIALIZED KW_VIEW tableName KW_REBUILD -> ^(TOK_REBUILD_MATERIALIZED_VIEW tableName) | KW_ALTER KW_INDEX alterIndexStatementSuffix -> alterIndexStatementSuffix | KW_ALTER (KW_DATABASE|KW_SCHEMA) alterDatabaseStatementSuffix -> alterDatabaseStatementSuffix ; @@ -1721,6 +1726,22 @@ createViewStatement ) ; +createMaterializedViewStatement +@init { + pushMsg("create materialized view statement", state); +} +@after { popMsg(state); } + : KW_CREATE KW_MATERIALIZED KW_VIEW (ifNotExists)? name=tableName + tableComment? tablePropertiesPrefixed? + KW_AS selectStatementWithCTE + -> ^(TOK_CREATE_MATERIALIZED_VIEW $name + ifNotExists? + tableComment? + tablePropertiesPrefixed? + selectStatementWithCTE + ) + ; + viewPartition @init { pushMsg("view partition specification", state); } @after { popMsg(state); } @@ -1734,6 +1755,12 @@ dropViewStatement : KW_DROP KW_VIEW ifExists? viewName -> ^(TOK_DROPVIEW viewName ifExists?) ; +dropMaterializedViewStatement +@init { pushMsg("drop materialized view statement", state); } +@after { popMsg(state); } + : KW_DROP KW_MATERIALIZED KW_VIEW ifExists? viewName -> ^(TOK_DROPTABLE viewName ifExists?) + ; + showFunctionIdentifier @init { pushMsg("identifier for show function statement", state); } @after { popMsg(state); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 1a9b42b..e7ca69f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -206,7 +206,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { ErrorMsg.OFFLINE_TABLE_OR_PARTITION.getMsg(":Table " + ts.tableName)); } - if (ts.tableHandle.isView()) { + if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } if (ts.tableHandle.isNonNative()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 5872e8e..cfc2698 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; import org.apache.hadoop.hive.ql.plan.LoadFileDesc; @@ -106,6 +107,7 @@ private AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc createTableDesc; + private CreateViewDesc createViewDesc; public ParseContext() { @@ -113,20 +115,14 @@ public ParseContext() { /** * @param conf - * @param qb - * current QB - * @param ast - * current parse tree * @param opToPartPruner * map from table scan operator to partition pruner * @param opToPartList * @param topOps * list of operators for the top query - * @param opParseCtx - * operator parse context - contains a mapping from operator to - * operator parse state (row resolver etc.) * @param joinOps * context needed join processing (map join specifically) + * @param smbMapJoinOps * @param loadTableWork * list of destination tables being loaded * @param loadFileWork @@ -138,13 +134,19 @@ public ParseContext() { * @param destTableId * @param listMapJoinOpsNoReducer * list of map join operators with no reducer - * @param groupOpToInputTables * @param prunedPartitions * @param opToSamplePruner * operator to sample pruner map * @param globalLimitCtx * @param nameToSplitSample * @param rootTasks + * @param opToPartToSkewedPruner + * @param viewAliasToInput + * @param reduceSinkOperatorsAddedByEnforceBucketingSorting + * @param analyzeRewrite + * @param createTableDesc + * @param queryProperties + * @param createViewDesc */ public ParseContext( HiveConf conf, @@ -165,7 +167,7 @@ public ParseContext( Map viewAliasToInput, List reduceSinkOperatorsAddedByEnforceBucketingSorting, AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc, - QueryProperties queryProperties) { + QueryProperties queryProperties, CreateViewDesc createViewDesc) { this.conf = conf; this.opToPartPruner = opToPartPruner; this.opToPartList = opToPartList; @@ -192,6 +194,7 @@ public ParseContext( this.analyzeRewrite = analyzeRewrite; this.createTableDesc = createTableDesc; this.queryProperties = queryProperties; + this.createViewDesc = createViewDesc; } /** @@ -531,4 +534,7 @@ public void setCreateTable(CreateTableDesc createTableDesc) { this.createTableDesc = createTableDesc; } + public CreateViewDesc getCreateViewDesc() { + return createViewDesc; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 7f4d0ff..ef149b1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; /** * Implementation of the query block. @@ -56,6 +57,9 @@ private CreateTableDesc tblDesc = null; // table descriptor of the final private CreateTableDesc directoryDesc = null ; + // If this is a materialized view, this stores the view descriptor + private CreateViewDesc viewDesc; + // used by PTFs /* * This map maintains the PTFInvocationSpec for each PTF chain invocation in this QB. @@ -387,4 +391,15 @@ public QBSubQuery getHavingClauseSubQueryPredicate() { return havingClauseSubQueryPredicate; } + public CreateViewDesc getViewDesc() { + return viewDesc; + } + + public void setViewDesc(CreateViewDesc viewDesc) { + this.viewDesc = viewDesc; + } + + public boolean isMaterializedView() { + return viewDesc != null && viewDesc.isMaterialized(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 30c87ad..82b4493 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -410,7 +410,7 @@ public ParseContext getParseContext() { listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, - analyzeRewrite, tableDesc, queryProperties); + analyzeRewrite, tableDesc, queryProperties, createVwDesc); } @SuppressWarnings("nls") @@ -1716,7 +1716,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException switch (ast.getToken().getType()) { case HiveParser.TOK_TAB: { TableSpec ts = new TableSpec(db, conf, ast); - if (ts.tableHandle.isView()) { + if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } @@ -1756,7 +1756,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException if ((!qb.getParseInfo().getIsSubQ()) && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { - if (qb.isCTAS()) { + if (qb.isCTAS() || qb.isMaterializedView()) { qb.setIsQuery(false); ctx.setResDir(null); ctx.setResFile(null); @@ -4054,12 +4054,10 @@ public static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, /** * Returns the GenericUDAFInfo struct for the aggregation. * - * @param aggName - * The name of the UDAF. + * @param evaluator + * @param emode * @param aggParameters * The exprNodeDesc of the original parameters - * @param aggTree - * The ASTNode node of the UDAF in the query. * @return GenericUDAFInfo * @throws SemanticException * when the UDAF is not found or has problems. @@ -4365,6 +4363,9 @@ private void processGroupingSetReduceSinkOperator(RowResolver reduceSinkInputRow * Generate the GroupByOperator for the Query Block (parseInfo.getXXX(dest)). * The new GroupByOperator will be a child of the reduceSinkOperatorInfo. * + * @param parseInfo + * @param dest + * @param reduceSinkOperatorInfo * @param mode * The mode of the aggregation (MERGEPARTIAL, PARTIAL2) * @param genericUDAFEvaluators @@ -4374,7 +4375,7 @@ private void processGroupingSetReduceSinkOperator(RowResolver reduceSinkInputRow * list of grouping sets * @param groupingSetsPresent * whether grouping sets are present in this query - * @param groupingSetsConsumedCurrentMR + * @param groupingSetsNeedAdditionalMRJob * whether grouping sets are consumed by this group by * @return the new GroupByOperator */ @@ -6456,9 +6457,13 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // table command rather than taking the default value List field_schemas = null; CreateTableDesc tblDesc = qb.getTableDesc(); + CreateViewDesc viewDesc = qb.getViewDesc(); if (tblDesc != null) { field_schemas = new ArrayList(); destTableIsTemporary = tblDesc.isTemporary(); + } else if (viewDesc != null) { + field_schemas = new ArrayList(); + destTableIsTemporary = false; } boolean first = true; @@ -6511,6 +6516,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // update the create table descriptor with the resulting schema. if (tblDesc != null) { tblDesc.setCols(new ArrayList(field_schemas)); + } else if (viewDesc != null) { + viewDesc.setSchema(new ArrayList(field_schemas)); } boolean isDestTempFile = true; @@ -6522,11 +6529,13 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE); - loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, dest_path, isDfsDir, cols, + loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols, colTypes)); if (tblDesc == null) { - if (qb.getIsQuery()) { + if (viewDesc != null) { + table_desc = PlanUtils.getTableDesc(viewDesc, cols, colTypes); + } else if (qb.getIsQuery()) { String fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); table_desc = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat); } else { @@ -9910,7 +9919,8 @@ private Operator genLateralViewPlan(QB qb, Operator op, ASTNode lateralViewTree) * * @param source * @param dest - * @param outputColNames + * @param colExprMap + * @param outputInternalColNames * - a list to which the new internal column names will be added, in * the same order as in the dest row resolver */ @@ -10007,10 +10017,12 @@ boolean genResolvedParseTree(ASTNode ast, PlannerContext plannerCtx) throws Sema } // 3. analyze create view command - if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW - || (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { + if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || + ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW || + ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW || + (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && + ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { child = analyzeCreateView(ast, qb); - SessionState.get().setCommandType(HiveOperation.CREATEVIEW); if (child == null) { return false; } @@ -10067,7 +10079,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, - analyzeRewrite, tableDesc, queryProperties); + analyzeRewrite, tableDesc, queryProperties, createVwDesc); // 5. Take care of view creation if (createVwDesc != null) { @@ -10075,20 +10087,22 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce // validate the create view statement at this point, the createVwDesc gets // all the information for semanticcheck - validateCreateView(createVwDesc); + validateCreateView(); - // Since we're only creating a view (not executing it), we don't need to - // optimize or translate the plan (and in fact, those procedures can - // interfere with the view creation). So skip the rest of this method. - ctx.setResDir(null); - ctx.setResFile(null); + if (!createVwDesc.isMaterialized()) { + // Since we're only creating a view (not executing it), we don't need to + // optimize or translate the plan (and in fact, those procedures can + // interfere with the view creation). So skip the rest of this method. + ctx.setResDir(null); + ctx.setResFile(null); - try { - PlanUtils.addInputsForView(pCtx); - } catch (HiveException e) { - throw new SemanticException(e); + try { + PlanUtils.addInputsForView(pCtx); + } catch (HiveException e) { + throw new SemanticException(e); + } + return; } - return; } // 6. Generate table access stats if required @@ -10210,6 +10224,11 @@ private void enforceScanLimits(ParseContext pCtx, FetchTask fTask) private void saveViewDefinition() throws SemanticException { + if (createVwDesc.isMaterialized() && createVwDesc.getOrReplace()) { + // This is a rebuild, there's nothing to do here. + return; + } + // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List derivedSchema = @@ -10315,7 +10334,10 @@ private void saveViewDefinition() throws SemanticException { } createVwDesc.setSchema(derivedSchema); - createVwDesc.setViewExpandedText(expandedText); + if (!createVwDesc.isMaterialized()) { + // materialized views don't store the expanded text as they won't be rewritten at query time. + createVwDesc.setViewExpandedText(expandedText); + } } static List convertRowSchemaToViewSchema(RowResolver rr) throws SemanticException { @@ -10605,7 +10627,7 @@ public RowResolver getRowResolver(Operator opt) { * Add default properties for table property. If a default parameter exists * in the tblProp, the value in tblProp will be kept. * - * @param table + * @param tblProp * property map * @return Modified table property map */ @@ -10670,8 +10692,7 @@ ASTNode analyzeCreateTable( RowFormatParams rowFormatParams = new RowFormatParams(); StorageFormat storageFormat = new StorageFormat(conf); - LOG.info("Creating table " + dbDotTab + " position=" - + ast.getCharPositionInLine()); + LOG.info("Creating table " + dbDotTab + " position=" + ast.getCharPositionInLine()); int numCh = ast.getChildCount(); /* @@ -10712,6 +10733,7 @@ ASTNode analyzeCreateTable( command_type = CTLT; } break; + case HiveParser.TOK_QUERY: // CTAS if (command_type == CTLT) { throw new SemanticException(ErrorMsg.CTAS_CTLT_COEXISTENCE.getMsg()); @@ -10796,7 +10818,7 @@ ASTNode analyzeCreateTable( } } - storageFormat.fillDefaultStorageFormat(isExt); + storageFormat.fillDefaultStorageFormat(isExt, false); if ((command_type == CTAS) && (storageFormat.getStorageHandler() != null)) { throw new SemanticException(ErrorMsg.CREATE_NON_NATIVE_AS.getMsg()); @@ -10909,6 +10931,7 @@ ASTNode analyzeCreateTable( SessionState.get().setCommandType(HiveOperation.CREATETABLE_AS_SELECT); return selectStmt; + default: throw new SemanticException("Unrecognized command."); } @@ -10924,8 +10947,7 @@ private void addDbAndTabToOutputs(String[] qualifiedTabName, TableType type) thr outputs.add(new WriteEntity(t, WriteEntity.WriteType.DDL_NO_LOCK)); } - private ASTNode analyzeCreateView(ASTNode ast, QB qb) - throws SemanticException { + private ASTNode analyzeCreateView(ASTNode ast, QB qb) throws SemanticException { String[] qualTabName = getQualifiedTableName((ASTNode) ast.getChild(0)); String dbDotTable = getDotName(qualTabName); List cols = null; @@ -10936,6 +10958,14 @@ private ASTNode analyzeCreateView(ASTNode ast, QB qb) ASTNode selectStmt = null; Map tblProps = null; List partColNames = null; + boolean isRebuild = ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW; + boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW + || isRebuild; + StorageFormat storageFormat = null; + if (isMaterialized) { + storageFormat = new StorageFormat(conf); + storageFormat.fillDefaultStorageFormat(false, true); + } LOG.info("Creating view " + dbDotTable + " position=" + ast.getCharPositionInLine()); @@ -10979,15 +11009,36 @@ private ASTNode analyzeCreateView(ASTNode ast, QB qb) orReplace = true; } - createVwDesc = new CreateViewDesc( - dbDotTable, cols, comment, tblProps, partColNames, - ifNotExists, orReplace, isAlterViewAs); - unparseTranslator.enable(); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - createVwDesc), conf)); + if (isMaterialized) { + createVwDesc = new CreateViewDesc(dbDotTable, cols, comment, tblProps, partColNames, + ifNotExists, orReplace || isRebuild, isAlterViewAs, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), storageFormat.getSerde()); + addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW); + SessionState.get().setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); + qb.setViewDesc(createVwDesc); + } else { + createVwDesc = new CreateViewDesc(dbDotTable, cols, comment, tblProps, partColNames, + ifNotExists, orReplace, isAlterViewAs); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + createVwDesc), conf)); + + addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); + SessionState.get().setCommandType(HiveOperation.CREATEVIEW); + } - addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); + if (isRebuild) { + // We need to go lookup the table and get the select statement and then parse it. + try { + Table tab = db.getTable(qualTabName[0], qualTabName[1]); + String viewText = tab.getViewOriginalText(); + ParseDriver pd = new ParseDriver(); + ASTNode tree = pd.parse(viewText, ctx, false); + selectStmt = ParseUtils.findRootNonNullToken(tree); + } catch (Exception e) { + throw new SemanticException(e); + } + } return selectStmt; } @@ -10998,7 +11049,7 @@ CreateViewDesc getCreateViewDesc() { // validate the create view statement // the statement could be CREATE VIEW, REPLACE VIEW, or ALTER VIEW AS SELECT // check semantic conditions - private void validateCreateView(CreateViewDesc createVwDesc) + private void validateCreateView() throws SemanticException { try { Table oldView = getTable(createVwDesc.getViewName(), false); @@ -11027,8 +11078,21 @@ private void validateCreateView(CreateViewDesc createVwDesc) //replace view if (createVwDesc.getOrReplace() && oldView != null) { + // Don't allow swapping between virtual and materialized view in replace + if (oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && createVwDesc.isMaterialized()) { + throw new SemanticException(ErrorMsg.REPLACE_VIEW_WITH_MATERIALIZED, + oldView.getTableName()); + } + + if (oldView.getTableType().equals(TableType.MATERIALIZED_VIEW) && + !createVwDesc.isMaterialized()) { + throw new SemanticException(ErrorMsg.REPLACE_MATERIALIZED_WITH_VIEW, + oldView.getTableName()); + } + // Existing table is not a view - if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW)) { + if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && + !oldView.getTableType().equals(TableType.MATERIALIZED_VIEW)) { String tableNotViewErrorMsg = "The following is an existing table, not a view: " + createVwDesc.getViewName(); @@ -11036,26 +11100,28 @@ private void validateCreateView(CreateViewDesc createVwDesc) ErrorMsg.EXISTING_TABLE_IS_NOT_VIEW.getMsg(tableNotViewErrorMsg)); } - // if old view has partitions, it could not be replaced - String partitionViewErrorMsg = - "The following view has partition, it could not be replaced: " + - createVwDesc.getViewName(); - try { - if ((createVwDesc.getPartCols() == null || - createVwDesc.getPartCols().isEmpty() || - !createVwDesc.getPartCols().equals(oldView.getPartCols())) && - !oldView.getPartCols().isEmpty() && - !db.getPartitions(oldView).isEmpty()) { + if (!createVwDesc.isMaterialized()) { + // if old view has partitions, it could not be replaced + String partitionViewErrorMsg = + "The following view has partition, it could not be replaced: " + + createVwDesc.getViewName(); + try { + if ((createVwDesc.getPartCols() == null || + createVwDesc.getPartCols().isEmpty() || + !createVwDesc.getPartCols().equals(oldView.getPartCols())) && + !oldView.getPartCols().isEmpty() && + !db.getPartitions(oldView).isEmpty()) { + throw new SemanticException( + ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + } + } catch (HiveException e) { throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); } - } catch (HiveException e) { - throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); } } } catch (HiveException e) { - throw new SemanticException(e.getMessage()); + throw new SemanticException(e.getMessage(), e); } } @@ -12146,6 +12212,7 @@ private void copyInfoToQueryProperties(QueryProperties queryProperties) { queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() && !qb.getParseInfo().getDestToOrderBy().isEmpty()); queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit()); + queryProperties.setMaterializedView(qb.getViewDesc() != null); } } private void warn(String msg) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 48aca4d..d3b955c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -104,13 +104,22 @@ protected void processStorageFormat(String name) throws SemanticException { } } - protected void fillDefaultStorageFormat(boolean isExternal) throws SemanticException { + protected void fillDefaultStorageFormat(boolean isExternal, boolean isMaterializedView) + throws SemanticException { if ((inputFormat == null) && (storageHandler == null)) { - String defaultFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT); - String defaultManagedFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTMANAGEDFILEFORMAT); + String defaultFormat; + String defaultManagedFormat; + if (isMaterializedView) { + defaultFormat = defaultManagedFormat = + HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMATERIALIZEDVIEWFILEFORMAT); + serde = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMATERIALIZEDVIEWSERDE); + } else { + defaultFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT); + defaultManagedFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTMANAGEDFILEFORMAT); + } if (!isExternal && !"none".equals(defaultManagedFormat)) { - defaultFormat = defaultManagedFormat; + defaultFormat = defaultManagedFormat; } if (StringUtils.isBlank(defaultFormat)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index b4d5382..7c32cc5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.LoadFileDesc; @@ -166,18 +167,25 @@ public void compile(final ParseContext pCtx, final List outIter = outputs.iterator(); - while (outIter.hasNext()) { - switch (outIter.next().getType()) { - case DFS_DIR: - case LOCAL_DIR: - outIter.remove(); - break; - default: - break; - } - } Task crtTblTask = TaskFactory.get(new DDLWork( inputs, outputs, crtTblDesc), conf); - - // find all leaf tasks and make the DDLTask as a dependent task of all of - // them - HashSet> leaves = new LinkedHashSet>(); - getLeafTasks(rootTasks, leaves); - assert (leaves.size() > 0); - for (Task task : leaves) { - if (task instanceof StatsTask) { - // StatsTask require table to already exist - for (Task parentOfStatsTask : task.getParentTasks()) { - parentOfStatsTask.addDependentTask(crtTblTask); - } - for (Task parentOfCrtTblTask : crtTblTask.getParentTasks()) { - parentOfCrtTblTask.removeDependentTask(task); - } - crtTblTask.addDependentTask(task); - } else { - task.addDependentTask(crtTblTask); - } - } + patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtTblTask); + } else if (pCtx.getQueryProperties().isMaterializedView()) { + // generate a DDL task and make it a dependent task of the leaf + CreateViewDesc viewDesc = pCtx.getCreateViewDesc(); + Task crtViewTask = TaskFactory.get(new DDLWork( + inputs, outputs, viewDesc), conf); + patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtViewTask); } if (globalLimitCtx.isEnable() && pCtx.getFetchTask() != null) { @@ -295,6 +277,44 @@ public void compile(final ParseContext pCtx, final List> rootTasks, + final HashSet outputs, + Task createTask) { + // clear the mapredWork output file from outputs for CTAS + // DDLWork at the tail of the chain will have the output + Iterator outIter = outputs.iterator(); + while (outIter.hasNext()) { + switch (outIter.next().getType()) { + case DFS_DIR: + case LOCAL_DIR: + outIter.remove(); + break; + default: + break; + } + } + + // find all leaf tasks and make the DDLTask as a dependent task of all of + // them + HashSet> leaves = + new LinkedHashSet>(); + getLeafTasks(rootTasks, leaves); + assert (leaves.size() > 0); + for (Task task : leaves) { + if (task instanceof StatsTask) { + // StatsTask require table to already exist + for (Task parentOfStatsTask : task.getParentTasks()) { + parentOfStatsTask.addDependentTask(createTask); + } + for (Task parentOfCrtTblTask : createTask.getParentTasks()) { + parentOfCrtTblTask.removeDependentTask(task); + } + createTask.addDependentTask(task); + } else { + task.addDependentTask(createTask); + } + } + } /** * A helper function to generate a column stats task on top of map-red task. The column stats @@ -304,7 +324,11 @@ public void compile(final ParseContext pCtx, final List loadTableWork, @@ -399,7 +423,8 @@ public ParseContext getParseContext(ParseContext pCtx, List partCols = mTable.getPartCols(); List bucketingCols = mTable.getBucketCols(); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java index a5cf076..8567217 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java @@ -46,6 +46,10 @@ private boolean ifNotExists; private boolean orReplace; private boolean isAlterViewAs; + private boolean isMaterialized; + private String inputFormat; // only used for materialized views + private String outputFormat; // only used for materialized views + private String serde; // only used for materialized views /** * For serialization only. @@ -53,6 +57,17 @@ public CreateViewDesc() { } + /** + * Used to create a virtual view descriptor. + * @param viewName + * @param schema + * @param comment + * @param tblProps + * @param partColNames + * @param ifNotExists + * @param orReplace + * @param isAlterViewAs + */ public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, List partColNames, boolean ifNotExists, @@ -65,6 +80,39 @@ public CreateViewDesc(String viewName, List schema, this.ifNotExists = ifNotExists; this.orReplace = orReplace; this.isAlterViewAs = isAlterViewAs; + this.isMaterialized = false; + } + + /** + * Used to create a materialized view descriptor + * @param viewName + * @param schema + * @param comment + * @param tblProps + * @param partColNames + * @param ifNotExists + * @param orReplace + * @param isAlterViewAs + * @param inputFormat + * @param outputFormat + * @param serde + */ + public CreateViewDesc(String viewName, List schema, String comment, + Map tblProps, List partColNames, + boolean ifNotExists, boolean orReplace, boolean isAlterViewAs, + String inputFormat, String outputFormat, String serde) { + this.viewName = viewName; + this.schema = schema; + this.tblProps = tblProps; + this.partColNames = partColNames; + this.comment = comment; + this.ifNotExists = ifNotExists; + this.orReplace = orReplace; + this.isAlterViewAs = isAlterViewAs; + this.isMaterialized = true; + this.inputFormat = inputFormat; + this.outputFormat = outputFormat; + this.serde = serde; } @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -172,4 +220,20 @@ public boolean getIsAlterViewAs() { public void setIsAlterViewAs(boolean isAlterViewAs) { this.isAlterViewAs = isAlterViewAs; } + + public boolean isMaterialized() { + return isMaterialized; + } + + public String getInputFormat() { + return inputFormat; + } + + public String getOutputFormat() { + return outputFormat; + } + + public String getSerde() { + return serde; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index 75cdf16..ff167e3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -75,6 +75,8 @@ CREATEMACRO("CREATEMACRO", null, null), DROPMACRO("DROPMACRO", null, null), CREATEVIEW("CREATEVIEW", new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE}), + CREATE_MATERIALIZED_VIEW("CREATEVIEW", new Privilege[]{Privilege.SELECT}, new + Privilege[]{Privilege.CREATE}), DROPVIEW("DROPVIEW", null, new Privilege[]{Privilege.DROP}), CREATEINDEX("CREATEINDEX", null, null), DROPINDEX("DROPINDEX", null, null), diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java index 0226278..23e5532 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java @@ -42,15 +42,19 @@ public LoadFileDesc() { } - public LoadFileDesc(final CreateTableDesc createTableDesc, final Path sourcePath, - final Path targetDir, - final boolean isDfsDir, final String columns, final String columnTypes) { + public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, + final Path sourcePath, final Path targetDir, final boolean isDfsDir, + final String columns, final String columnTypes) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes); if (createTableDesc != null && createTableDesc.getDatabaseName() != null && createTableDesc.getTableName() != null) { destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc .getDatabaseName() + ".") + createTableDesc.getTableName(); + } else if (createViewDesc != null) { + // The work is already done in analyzeCreateView to assure that the view name is fully + // qualified. + destinationCreateTable = createViewDesc.getViewName(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 87a2548..07d428d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -371,6 +371,47 @@ public static TableDesc getTableDesc(CreateTableDesc crtTblDesc, String cols, return ret; } + /** + * Generate a table descriptor from a createViewDesc. + */ + public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, String colTypes) { + TableDesc ret; + + try { + Class serdeClass = JavaUtils.loadClass(crtViewDesc.getSerde()); + ret = getTableDesc(serdeClass, new String(LazySerDeParameters.DefaultSeparators), cols, + colTypes, false, false); + + // set other table properties + /* + TODO - I don't think I need any of this + Properties properties = ret.getProperties(); + + if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, + crtTblDesc.getTableName()); + } + + if (crtTblDesc.getTblProps() != null) { + properties.putAll(crtTblDesc.getTblProps()); + } + */ + + // replace the default input & output file format with those found in + // crtTblDesc + Class inClass = + (Class)JavaUtils.loadClass(crtViewDesc.getInputFormat()); + Class outClass = + (Class)JavaUtils.loadClass(crtViewDesc.getOutputFormat()); + + ret.setInputFileFormatClass(inClass); + ret.setOutputFileFormatClass(outClass); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e); + } + return ret; + } + /** * Generate the table descriptor of MetadataTypedColumnsetSerDe with the * separatorCode. MetaDataTypedColumnsetSerDe is used because LazySimpleSerDe diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index b79c080..39c741a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -77,6 +77,7 @@ CREATEMACRO, DROPMACRO, CREATEVIEW, + CREATE_MATERIALIZED_VIEW, DROPVIEW, CREATEINDEX, DROPINDEX, diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index d43eee4..c48a52f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -314,6 +314,10 @@ public HivePrivilegeObjectType getObjectType() { new PrivRequirement(SEL_GRANT_AR, IOType.INPUT), new PrivRequirement(OWNER_PRIV_AR, HivePrivilegeObjectType.DATABASE))); + op2Priv.put(HiveOperationType.CREATE_MATERIALIZED_VIEW, PrivRequirement.newPrivRequirementList( + new PrivRequirement(SEL_GRANT_AR, IOType.INPUT), + new PrivRequirement(OWNER_PRIV_AR, HivePrivilegeObjectType.DATABASE))); + op2Priv.put(HiveOperationType.SHOWFUNCTIONS, PrivRequirement.newIOPrivRequirement (null, null)); op2Priv.put(HiveOperationType.SHOWINDEXES, PrivRequirement.newIOPrivRequirement diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_grant.q ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_grant.q new file mode 100644 index 0000000..079baff --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_grant.q @@ -0,0 +1,15 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvcng_gtable (a int, b varchar(256), c decimal(10,2)); + +insert into amvcng_gtable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +grant select on table amvcng_gtable to user user2; + +set user.name=user2; + +create materialized view amvcng_gmat_view as select a, c from amvcng_gtable; diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_select_perm.q ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_select_perm.q new file mode 100644 index 0000000..4de525c --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_create_no_select_perm.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvnsp_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + + +set user.name=user2; + +create materialized view amvnsp_mat_view as select a, c from amvnsp_table; diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_drop_other.q ql/src/test/queries/clientnegative/authorization_materialized_view_drop_other.q new file mode 100644 index 0000000..c9a5930 --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_drop_other.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvdo_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvdo_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvdo_mat_view as select a, c from amvdo_table; + +set user.name=user2; +drop materialized view amvdo_mat_view; diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_no_select_perm.q ql/src/test/queries/clientnegative/authorization_materialized_view_no_select_perm.q new file mode 100644 index 0000000..8428152 --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_no_select_perm.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvnsp_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvnsp_mat_view as select a, c from amvnsp_table; + +set user.name=user2; +select * from amvnsp_mat_view; diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_no_grant.q ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_no_grant.q new file mode 100644 index 0000000..a2e7d38 --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_no_grant.q @@ -0,0 +1,20 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvrng_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +grant select on table amvrng_table to user user2 with grant option; + +set user.name=user2; +create materialized view amvrng_mat_view as select a, c from amvrng_table; + +set user.name=user1; +revoke grant option for select on table amvrng_table from user user2; + +set user.name=user2; +alter materialized view amvrng_mat_view rebuild; diff --git ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_other.q ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_other.q new file mode 100644 index 0000000..7c2d145 --- /dev/null +++ ql/src/test/queries/clientnegative/authorization_materialized_view_rebuild_other.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvro_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvro_mat_view as select a, c from amvro_table; + +set user.name=user2; +alter materialized view amvro_mat_view rebuild; diff --git ql/src/test/queries/clientnegative/delete_materialized_view.q ql/src/test/queries/clientnegative/delete_materialized_view.q new file mode 100644 index 0000000..f557df9 --- /dev/null +++ ql/src/test/queries/clientnegative/delete_materialized_view.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table dmv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view dmv_mat_view as select a, b, c from dmv_basetable; + +delete from dmv_mat_view where b = 'fred'; diff --git ql/src/test/queries/clientnegative/delete_view.q ql/src/test/queries/clientnegative/delete_view.q new file mode 100644 index 0000000..c9dc985 --- /dev/null +++ ql/src/test/queries/clientnegative/delete_view.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table dv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create view dv_view as select a, b, c from dv_basetable; + +delete from dv_view where b = 'fred'; diff --git ql/src/test/queries/clientnegative/insert_materialized_view.q ql/src/test/queries/clientnegative/insert_materialized_view.q new file mode 100644 index 0000000..2daae52 --- /dev/null +++ ql/src/test/queries/clientnegative/insert_materialized_view.q @@ -0,0 +1,6 @@ +create table imv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view imv_mat_view as select a, b, c from imv_basetable; + +insert into imv_mat_view values (1, 'fred', 3.14); diff --git ql/src/test/queries/clientnegative/load_materialized_view.q ql/src/test/queries/clientnegative/load_materialized_view.q new file mode 100644 index 0000000..a8d0dfd --- /dev/null +++ ql/src/test/queries/clientnegative/load_materialized_view.q @@ -0,0 +1,7 @@ +create table lmv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view lmv_mat_view as select a, b, c from lmv_basetable; + +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE lmv_mat_view; + diff --git ql/src/test/queries/clientnegative/replace_materialized_view_with_view.q ql/src/test/queries/clientnegative/replace_materialized_view_with_view.q new file mode 100644 index 0000000..7c972f5 --- /dev/null +++ ql/src/test/queries/clientnegative/replace_materialized_view_with_view.q @@ -0,0 +1,8 @@ +create table rmvwv_basetable (a int, b varchar(256), c decimal(10,2)); + +insert into rmvwv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view rmvwv_mat_view as select a, b, c from rmvwv_basetable; + +create or replace view rmvwv_mat_view as select a, c from rmvwv_basetable; + diff --git ql/src/test/queries/clientnegative/update_materialized_view.q ql/src/test/queries/clientnegative/update_materialized_view.q new file mode 100644 index 0000000..8245ef0 --- /dev/null +++ ql/src/test/queries/clientnegative/update_materialized_view.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table umv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view umv_mat_view as select a, b, c from umv_basetable; + +update umv_mat_view set b = 'joe' where b = 'fred'; diff --git ql/src/test/queries/clientnegative/update_view.q ql/src/test/queries/clientnegative/update_view.q new file mode 100644 index 0000000..3a54ccb --- /dev/null +++ ql/src/test/queries/clientnegative/update_view.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table uv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create view uv_view as select a, b, c from uv_basetable; + +update uv_view set b = 'joe' where b = 'fred'; diff --git ql/src/test/queries/clientpositive/authorization_materialized_view_sqlstd.q ql/src/test/queries/clientpositive/authorization_materialized_view_sqlstd.q new file mode 100644 index 0000000..87a53e9 --- /dev/null +++ ql/src/test/queries/clientpositive/authorization_materialized_view_sqlstd.q @@ -0,0 +1,58 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvs_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvs_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvs_mat_view as select a, c from amvs_table; + +show grant user user1 on table amvs_mat_view; + +grant select on amvs_mat_view to user user2; + +set user.name=user2; +show grant user user2 on table amvs_mat_view; +select * from amvs_mat_view; + +set user.name=user3; +show grant user user3 on table amvs_mat_view; + + +set user.name=hive_admin_user; +set role admin; +show grant on table amvs_mat_view; + +set user.name=user1; +revoke select on table amvs_mat_view from user user2; +set user.name=user2; +show grant user user2 on table amvs_mat_view; + +set user.name=hive_admin_user; +set role ADMIN; +create role role_v; +grant role_v to user user4 ; +show role grant user user4; +show roles; + +grant all on table amvs_mat_view to role role_v; +show grant role role_v on table amvs_mat_view; +show grant user user4 on table amvs_mat_view; +select * from amvs_mat_view; + +set user.name=user1; +alter materialized view amvs_mat_view rebuild; + +grant select on table amvs_table to user user2 with grant option; +set user.name=user2; +create materialized view amvs_mat_view2 as select a, b from amvs_table; + +select * from amvs_mat_view2; + +drop materialized view amvs_mat_view2; + +set user.name=hive_admin_user; +set role ADMIN; diff --git ql/src/test/queries/clientpositive/authorization_view_sqlstd.q ql/src/test/queries/clientpositive/authorization_view_sqlstd.q index 85d3ca3..55e93b3 100644 --- ql/src/test/queries/clientpositive/authorization_view_sqlstd.q +++ ql/src/test/queries/clientpositive/authorization_view_sqlstd.q @@ -7,6 +7,7 @@ set user.name=user1; -- Test view authorization , and 'show grant' variants create table t1(i int, j int, k int); +grant select on t1 to user user2 with grant option; show grant user user1 on table t1; -- protecting certain columns @@ -25,6 +26,8 @@ grant insert on table vt1 to user user3; set user.name=user2; show grant user user2 on table vt1; +create view vt3 as select i,k from t1; + set user.name=user3; show grant user user3 on table vt1; diff --git ql/src/test/queries/clientpositive/create_materialized_view.q ql/src/test/queries/clientpositive/create_materialized_view.q new file mode 100644 index 0000000..57f8bb7 --- /dev/null +++ ql/src/test/queries/clientpositive/create_materialized_view.q @@ -0,0 +1,31 @@ +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view as select a, b, c from cmv_basetable; + +select * from cmv_mat_view; + +create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable; + +select * from cmv_mat_view2; + +create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1; + +select * from cmv_mat_view3; + +create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a; + +select * from cmv_mat_view4; + +describe extended cmv_mat_view4; + +create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2)); + +insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8); + +create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e); + +select * from cmv_mat_view5; + +show tblproperties cmv_mat_view5; diff --git ql/src/test/queries/clientpositive/drop_materialized_view.q ql/src/test/queries/clientpositive/drop_materialized_view.q new file mode 100644 index 0000000..8897159 --- /dev/null +++ ql/src/test/queries/clientpositive/drop_materialized_view.q @@ -0,0 +1,7 @@ +create materialized view dmv_mat_view as select cint, cstring1 from alltypesorc where cint < 0; + +show table extended like dmv_mat_view; + +drop materialized view dmv_mat_view; + +show table extended like dmv_mat_view; diff --git ql/src/test/queries/clientpositive/rebuild_materialized_view.q ql/src/test/queries/clientpositive/rebuild_materialized_view.q new file mode 100644 index 0000000..c0a5530 --- /dev/null +++ ql/src/test/queries/clientpositive/rebuild_materialized_view.q @@ -0,0 +1,13 @@ +create table rmv_table (cint int, cstring1 string); + +insert into rmv_table values(1, 'fred'), (10, 'wilma'); + +create materialized view rmv_mat_view as select cint, cstring1 from rmv_table where cint < 10; + +select * from rmv_mat_view; + +insert into rmv_table values(2, 'barney'), (11, 'betty'); + +alter materialized view rmv_mat_view rebuild; + +select * from rmv_mat_view; diff --git ql/src/test/results/clientnegative/authorization_materialized_view_create_no_grant.q.out ql/src/test/results/clientnegative/authorization_materialized_view_create_no_grant.q.out new file mode 100644 index 0000000..8c6da0d --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_create_no_grant.q.out @@ -0,0 +1,26 @@ +PREHOOK: query: create table amvcng_gtable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvcng_gtable +POSTHOOK: query: create table amvcng_gtable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvcng_gtable +PREHOOK: query: insert into amvcng_gtable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvcng_gtable +POSTHOOK: query: insert into amvcng_gtable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvcng_gtable +POSTHOOK: Lineage: amvcng_gtable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvcng_gtable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvcng_gtable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: grant select on table amvcng_gtable to user user2 +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvcng_gtable +POSTHOOK: query: grant select on table amvcng_gtable to user user2 +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvcng_gtable +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvcng_gtable]] diff --git ql/src/test/results/clientnegative/authorization_materialized_view_create_no_select_perm.q.out ql/src/test/results/clientnegative/authorization_materialized_view_create_no_select_perm.q.out new file mode 100644 index 0000000..67d3474 --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_create_no_select_perm.q.out @@ -0,0 +1,20 @@ +PREHOOK: query: create table amvnsp_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvnsp_table +POSTHOOK: query: create table amvnsp_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvnsp_table +PREHOOK: query: insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvnsp_table +POSTHOOK: query: insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvnsp_table +POSTHOOK: Lineage: amvnsp_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvnsp_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvnsp_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvnsp_table]] diff --git ql/src/test/results/clientnegative/authorization_materialized_view_drop_other.q.out ql/src/test/results/clientnegative/authorization_materialized_view_drop_other.q.out new file mode 100644 index 0000000..d1aff33 --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_drop_other.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: create table amvdo_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvdo_table +POSTHOOK: query: create table amvdo_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvdo_table +PREHOOK: query: insert into amvdo_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvdo_table +POSTHOOK: query: insert into amvdo_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvdo_table +POSTHOOK: Lineage: amvdo_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvdo_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvdo_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view amvdo_mat_view as select a, c from amvdo_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvdo_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvdo_mat_view +POSTHOOK: query: create materialized view amvdo_mat_view as select a, c from amvdo_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvdo_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvdo_mat_view +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation DROPTABLE [[OBJECT OWNERSHIP] on Object [type=TABLE_OR_VIEW, name=default.amvdo_mat_view]] diff --git ql/src/test/results/clientnegative/authorization_materialized_view_no_select_perm.q.out ql/src/test/results/clientnegative/authorization_materialized_view_no_select_perm.q.out new file mode 100644 index 0000000..a3a3405 --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_no_select_perm.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: create table amvnsp_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvnsp_table +POSTHOOK: query: create table amvnsp_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvnsp_table +PREHOOK: query: insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvnsp_table +POSTHOOK: query: insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvnsp_table +POSTHOOK: Lineage: amvnsp_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvnsp_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvnsp_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view amvnsp_mat_view as select a, c from amvnsp_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvnsp_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvnsp_mat_view +POSTHOOK: query: create materialized view amvnsp_mat_view as select a, c from amvnsp_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvnsp_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvnsp_mat_view +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation QUERY [[SELECT] on Object [type=TABLE_OR_VIEW, name=default.amvnsp_mat_view]] diff --git ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_no_grant.q.out ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_no_grant.q.out new file mode 100644 index 0000000..dc7a58d --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_no_grant.q.out @@ -0,0 +1,42 @@ +PREHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvrng_table +PREHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvrng_table +POSTHOOK: Lineage: amvrng_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvrng_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvrng_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: grant select on table amvrng_table to user user2 with grant option +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: grant select on table amvrng_table to user user2 with grant option +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvrng_table +PREHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvrng_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvrng_mat_view +POSTHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvrng_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvrng_mat_view +PREHOOK: query: revoke grant option for select on table amvrng_table from user user2 +PREHOOK: type: REVOKE_PRIVILEGE +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: revoke grant option for select on table amvrng_table from user user2 +POSTHOOK: type: REVOKE_PRIVILEGE +POSTHOOK: Output: default@amvrng_table +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvrng_table]] diff --git ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_other.q.out ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_other.q.out new file mode 100644 index 0000000..9c25b52 --- /dev/null +++ ql/src/test/results/clientnegative/authorization_materialized_view_rebuild_other.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvro_table +POSTHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvro_table +PREHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvro_table +POSTHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvro_table +POSTHOOK: Lineage: amvro_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvro_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvro_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvro_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvro_mat_view +POSTHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvro_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvro_mat_view +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvro_table]] diff --git ql/src/test/results/clientnegative/delete_materialized_view.q.out ql/src/test/results/clientnegative/delete_materialized_view.q.out new file mode 100644 index 0000000..6013e1c --- /dev/null +++ ql/src/test/results/clientnegative/delete_materialized_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table dmv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dmv_basetable +POSTHOOK: query: create table dmv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dmv_basetable +PREHOOK: query: create materialized view dmv_mat_view as select a, b, c from dmv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@dmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@dmv_mat_view +POSTHOOK: query: create materialized view dmv_mat_view as select a, b, c from dmv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@dmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dmv_mat_view +FAILED: SemanticException [Error 10304]: You cannot update or delete records in a view diff --git ql/src/test/results/clientnegative/delete_view.q.out ql/src/test/results/clientnegative/delete_view.q.out new file mode 100644 index 0000000..3e72344 --- /dev/null +++ ql/src/test/results/clientnegative/delete_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table dv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dv_basetable +POSTHOOK: query: create table dv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dv_basetable +PREHOOK: query: create view dv_view as select a, b, c from dv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@dv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@dv_view +POSTHOOK: query: create view dv_view as select a, b, c from dv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@dv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dv_view +FAILED: SemanticException [Error 10304]: You cannot update or delete records in a view diff --git ql/src/test/results/clientnegative/insert_materialized_view.q.out ql/src/test/results/clientnegative/insert_materialized_view.q.out new file mode 100644 index 0000000..08b9a78 --- /dev/null +++ ql/src/test/results/clientnegative/insert_materialized_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table imv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@imv_basetable +POSTHOOK: query: create table imv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@imv_basetable +PREHOOK: query: create materialized view imv_mat_view as select a, b, c from imv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@imv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@imv_mat_view +POSTHOOK: query: create materialized view imv_mat_view as select a, b, c from imv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@imv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@imv_mat_view +FAILED: SemanticException [Error 10090]: A view cannot be used as target table for LOAD or INSERT diff --git ql/src/test/results/clientnegative/load_materialized_view.q.out ql/src/test/results/clientnegative/load_materialized_view.q.out new file mode 100644 index 0000000..4814365 --- /dev/null +++ ql/src/test/results/clientnegative/load_materialized_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table lmv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lmv_basetable +POSTHOOK: query: create table lmv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lmv_basetable +PREHOOK: query: create materialized view lmv_mat_view as select a, b, c from lmv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@lmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@lmv_mat_view +POSTHOOK: query: create materialized view lmv_mat_view as select a, b, c from lmv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@lmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lmv_mat_view +FAILED: SemanticException [Error 10090]: A view cannot be used as target table for LOAD or INSERT diff --git ql/src/test/results/clientnegative/replace_materialized_view_with_view.q.out ql/src/test/results/clientnegative/replace_materialized_view_with_view.q.out new file mode 100644 index 0000000..dac3538 --- /dev/null +++ ql/src/test/results/clientnegative/replace_materialized_view_with_view.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: create table rmvwv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@rmvwv_basetable +POSTHOOK: query: create table rmvwv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rmvwv_basetable +PREHOOK: query: insert into rmvwv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@rmvwv_basetable +POSTHOOK: query: insert into rmvwv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@rmvwv_basetable +POSTHOOK: Lineage: rmvwv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: rmvwv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: rmvwv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view rmvwv_mat_view as select a, b, c from rmvwv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@rmvwv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@rmvwv_mat_view +POSTHOOK: query: create materialized view rmvwv_mat_view as select a, b, c from rmvwv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@rmvwv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rmvwv_mat_view +FAILED: SemanticException [Error 10303]: Attempt to replace materialized view rmvwv_mat_view with view diff --git ql/src/test/results/clientnegative/update_materialized_view.q.out ql/src/test/results/clientnegative/update_materialized_view.q.out new file mode 100644 index 0000000..bee63a8 --- /dev/null +++ ql/src/test/results/clientnegative/update_materialized_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table umv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@umv_basetable +POSTHOOK: query: create table umv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@umv_basetable +PREHOOK: query: create materialized view umv_mat_view as select a, b, c from umv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@umv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@umv_mat_view +POSTHOOK: query: create materialized view umv_mat_view as select a, b, c from umv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@umv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@umv_mat_view +FAILED: SemanticException [Error 10304]: You cannot update or delete records in a view diff --git ql/src/test/results/clientnegative/update_view.q.out ql/src/test/results/clientnegative/update_view.q.out new file mode 100644 index 0000000..8a12665 --- /dev/null +++ ql/src/test/results/clientnegative/update_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table uv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@uv_basetable +POSTHOOK: query: create table uv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@uv_basetable +PREHOOK: query: create view uv_view as select a, b, c from uv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@uv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@uv_view +POSTHOOK: query: create view uv_view as select a, b, c from uv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@uv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@uv_view +FAILED: SemanticException [Error 10304]: You cannot update or delete records in a view diff --git ql/src/test/results/clientpositive/authorization_materialized_view_sqlstd.q.out ql/src/test/results/clientpositive/authorization_materialized_view_sqlstd.q.out new file mode 100644 index 0000000..b167702 --- /dev/null +++ ql/src/test/results/clientpositive/authorization_materialized_view_sqlstd.q.out @@ -0,0 +1,195 @@ +PREHOOK: query: create table amvs_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvs_table +POSTHOOK: query: create table amvs_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvs_table +PREHOOK: query: insert into amvs_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@amvs_table +POSTHOOK: query: insert into amvs_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@amvs_table +POSTHOOK: Lineage: amvs_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvs_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvs_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view amvs_mat_view as select a, c from amvs_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvs_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvs_mat_view +POSTHOOK: query: create materialized view amvs_mat_view as select a, c from amvs_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvs_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvs_mat_view +PREHOOK: query: show grant user user1 on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user user1 on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +default amvs_mat_view user1 USER DELETE true -1 user1 +default amvs_mat_view user1 USER INSERT true -1 user1 +default amvs_mat_view user1 USER SELECT true -1 user1 +default amvs_mat_view user1 USER UPDATE true -1 user1 +PREHOOK: query: grant select on amvs_mat_view to user user2 +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvs_mat_view +POSTHOOK: query: grant select on amvs_mat_view to user user2 +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvs_mat_view +PREHOOK: query: show grant user user2 on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user user2 on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +default amvs_mat_view user2 USER SELECT false -1 user1 +PREHOOK: query: select * from amvs_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@amvs_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from amvs_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@amvs_mat_view +#### A masked pattern was here #### +1 10.3 +2 3.14 +2 172342.2 +3 978.76 +3 9.8 +PREHOOK: query: show grant user user3 on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user user3 on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +PREHOOK: query: set role admin +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role admin +POSTHOOK: type: SHOW_ROLES +PREHOOK: query: show grant on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +default amvs_mat_view user1 USER DELETE true -1 user1 +default amvs_mat_view user1 USER INSERT true -1 user1 +default amvs_mat_view user1 USER SELECT true -1 user1 +default amvs_mat_view user1 USER UPDATE true -1 user1 +default amvs_mat_view user2 USER SELECT false -1 user1 +PREHOOK: query: revoke select on table amvs_mat_view from user user2 +PREHOOK: type: REVOKE_PRIVILEGE +PREHOOK: Output: default@amvs_mat_view +POSTHOOK: query: revoke select on table amvs_mat_view from user user2 +POSTHOOK: type: REVOKE_PRIVILEGE +POSTHOOK: Output: default@amvs_mat_view +PREHOOK: query: show grant user user2 on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user user2 on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +PREHOOK: query: set role ADMIN +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role ADMIN +POSTHOOK: type: SHOW_ROLES +PREHOOK: query: create role role_v +PREHOOK: type: CREATEROLE +POSTHOOK: query: create role role_v +POSTHOOK: type: CREATEROLE +PREHOOK: query: grant role_v to user user4 +PREHOOK: type: GRANT_ROLE +POSTHOOK: query: grant role_v to user user4 +POSTHOOK: type: GRANT_ROLE +PREHOOK: query: show role grant user user4 +PREHOOK: type: SHOW_ROLE_GRANT +POSTHOOK: query: show role grant user user4 +POSTHOOK: type: SHOW_ROLE_GRANT +public false -1 +role_v false -1 hive_admin_user +PREHOOK: query: show roles +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: show roles +POSTHOOK: type: SHOW_ROLES +admin +public +role_v +PREHOOK: query: grant all on table amvs_mat_view to role role_v +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvs_mat_view +POSTHOOK: query: grant all on table amvs_mat_view to role role_v +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvs_mat_view +PREHOOK: query: show grant role role_v on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant role role_v on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +default amvs_mat_view role_v ROLE DELETE false -1 hive_admin_user +default amvs_mat_view role_v ROLE INSERT false -1 hive_admin_user +default amvs_mat_view role_v ROLE SELECT false -1 hive_admin_user +default amvs_mat_view role_v ROLE UPDATE false -1 hive_admin_user +PREHOOK: query: show grant user user4 on table amvs_mat_view +PREHOOK: type: SHOW_GRANT +POSTHOOK: query: show grant user user4 on table amvs_mat_view +POSTHOOK: type: SHOW_GRANT +PREHOOK: query: select * from amvs_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@amvs_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from amvs_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@amvs_mat_view +#### A masked pattern was here #### +1 10.3 +2 3.14 +2 172342.2 +3 978.76 +3 9.8 +PREHOOK: query: alter materialized view amvs_mat_view rebuild +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvs_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvs_mat_view +POSTHOOK: query: alter materialized view amvs_mat_view rebuild +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvs_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvs_mat_view +PREHOOK: query: grant select on table amvs_table to user user2 with grant option +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvs_table +POSTHOOK: query: grant select on table amvs_table to user user2 with grant option +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvs_table +PREHOOK: query: create materialized view amvs_mat_view2 as select a, b from amvs_table +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@amvs_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvs_mat_view2 +POSTHOOK: query: create materialized view amvs_mat_view2 as select a, b from amvs_table +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@amvs_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvs_mat_view2 +PREHOOK: query: select * from amvs_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@amvs_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from amvs_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@amvs_mat_view2 +#### A masked pattern was here #### +1 alfred +2 bob +2 bonnie +3 calvin +3 charlie +PREHOOK: query: drop materialized view amvs_mat_view2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@amvs_mat_view2 +PREHOOK: Output: default@amvs_mat_view2 +POSTHOOK: query: drop materialized view amvs_mat_view2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@amvs_mat_view2 +POSTHOOK: Output: default@amvs_mat_view2 +PREHOOK: query: set role ADMIN +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role ADMIN +POSTHOOK: type: SHOW_ROLES diff --git ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out index 54c4ce7..79ae929 100644 --- ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out +++ ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out @@ -10,6 +10,12 @@ create table t1(i int, j int, k int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@t1 +PREHOOK: query: grant select on t1 to user user2 with grant option +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@t1 +POSTHOOK: query: grant select on t1 to user user2 with grant option +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@t1 PREHOOK: query: show grant user user1 on table t1 PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user user1 on table t1 @@ -81,6 +87,16 @@ PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user user2 on table vt1 POSTHOOK: type: SHOW_GRANT default vt1 user2 USER SELECT false -1 user1 +PREHOOK: query: create view vt3 as select i,k from t1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t1 +PREHOOK: Output: database:default +PREHOOK: Output: default@vt3 +POSTHOOK: query: create view vt3 as select i,k from t1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vt3 PREHOOK: query: show grant user user3 on table vt1 PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user user3 on table vt1 @@ -127,11 +143,16 @@ PREHOOK: query: show grant user user2 on all PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user user2 on all POSTHOOK: type: SHOW_GRANT +default t1 user2 USER SELECT true -1 user1 default vt1 user2 USER SELECT false -1 user1 default vt2 user2 USER DELETE false -1 user1 default vt2 user2 USER INSERT false -1 user1 default vt2 user2 USER SELECT false -1 user1 default vt2 user2 USER UPDATE false -1 user1 +default vt3 user2 USER DELETE true -1 user1 +default vt3 user2 USER INSERT true -1 user1 +default vt3 user2 USER SELECT true -1 user1 +default vt3 user2 USER UPDATE true -1 user1 PREHOOK: query: revoke all on vt2 from user user2 PREHOOK: type: REVOKE_PRIVILEGE PREHOOK: Output: default@vt2 @@ -168,6 +189,11 @@ PREHOOK: query: show grant user user2 on all PREHOOK: type: SHOW_GRANT POSTHOOK: query: show grant user user2 on all POSTHOOK: type: SHOW_GRANT +default t1 user2 USER SELECT true -1 user1 +default vt3 user2 USER DELETE true -1 user1 +default vt3 user2 USER INSERT true -1 user1 +default vt3 user2 USER SELECT true -1 user1 +default vt3 user2 USER UPDATE true -1 user1 PREHOOK: query: -- grant privileges on roles for view, after next statement show grant user user3 on table vt1 PREHOOK: type: SHOW_GRANT diff --git ql/src/test/results/clientpositive/create_materialized_view.q.out ql/src/test/results/clientpositive/create_materialized_view.q.out new file mode 100644 index 0000000..c51357d --- /dev/null +++ ql/src/test/results/clientpositive/create_materialized_view.q.out @@ -0,0 +1,173 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: select * from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 alfred 10.3 +2 bob 3.14 +2 bonnie 172342.2 +3 calvin 978.76 +3 charlie 9.8 +PREHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: select * from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +1 10.3 +2 3.14 +2 172342.2 +3 978.76 +3 9.8 +PREHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view3 +POSTHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view3 +PREHOOK: query: select * from cmv_mat_view3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +2 bob 3.14 +2 bonnie 172342.2 +3 calvin 978.76 +3 charlie 9.8 +PREHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view4 +POSTHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view4 +PREHOOK: query: select * from cmv_mat_view4 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +1 10.3 +2 172345.34 +3 988.56 +PREHOOK: query: describe extended cmv_mat_view4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view4 +POSTHOOK: query: describe extended cmv_mat_view4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view4 +a int +_c1 decimal(20,2) + +#### A masked pattern was here #### +PREHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable2 +POSTHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable2 +PREHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@cmv_basetable2 +POSTHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@cmv_basetable2 +POSTHOOK: Lineage: cmv_basetable2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable2.e EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable2.f EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view5 +POSTHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view5 +PREHOOK: query: select * from cmv_mat_view5 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view5 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view5 +#### A masked pattern was here #### +1 alfred 4 10.3 100.3 +2 bob 4 3.14 6133 +2 bonnie 5 172342.2 172.2 +3 calvin 6 978.76 8.76 +3 charlie 17 9.8 13144339.8 +PREHOOK: query: show tblproperties cmv_mat_view5 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view5 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE false +key value +numFiles 1 +numRows -1 +rawDataSize -1 +totalSize 730 +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/drop_materialized_view.q.out ql/src/test/results/clientpositive/drop_materialized_view.q.out new file mode 100644 index 0000000..ed99a09 --- /dev/null +++ ql/src/test/results/clientpositive/drop_materialized_view.q.out @@ -0,0 +1,39 @@ +PREHOOK: query: create materialized view dmv_mat_view as select cint, cstring1 from alltypesorc where cint < 0 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@dmv_mat_view +POSTHOOK: query: create materialized view dmv_mat_view as select cint, cstring1 from alltypesorc where cint < 0 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dmv_mat_view +PREHOOK: query: show table extended like dmv_mat_view +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like dmv_mat_view +POSTHOOK: type: SHOW_TABLESTATUS +tableName:dmv_mat_view +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 cint, string cstring1} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:47146 +maxFileSize:47146 +minFileSize:47146 +#### A masked pattern was here #### + +PREHOOK: query: drop materialized view dmv_mat_view +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@dmv_mat_view +PREHOOK: Output: default@dmv_mat_view +POSTHOOK: query: drop materialized view dmv_mat_view +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@dmv_mat_view +POSTHOOK: Output: default@dmv_mat_view +PREHOOK: query: show table extended like dmv_mat_view +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like dmv_mat_view +POSTHOOK: type: SHOW_TABLESTATUS diff --git ql/src/test/results/clientpositive/rebuild_materialized_view.q.out ql/src/test/results/clientpositive/rebuild_materialized_view.q.out new file mode 100644 index 0000000..9bede0d --- /dev/null +++ ql/src/test/results/clientpositive/rebuild_materialized_view.q.out @@ -0,0 +1,67 @@ +PREHOOK: query: create table rmv_table (cint int, cstring1 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@rmv_table +POSTHOOK: query: create table rmv_table (cint int, cstring1 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rmv_table +PREHOOK: query: insert into rmv_table values(1, 'fred'), (10, 'wilma') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@rmv_table +POSTHOOK: query: insert into rmv_table values(1, 'fred'), (10, 'wilma') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@rmv_table +POSTHOOK: Lineage: rmv_table.cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: rmv_table.cstring1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: create materialized view rmv_mat_view as select cint, cstring1 from rmv_table where cint < 10 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@rmv_table +PREHOOK: Output: database:default +PREHOOK: Output: default@rmv_mat_view +POSTHOOK: query: create materialized view rmv_mat_view as select cint, cstring1 from rmv_table where cint < 10 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@rmv_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rmv_mat_view +PREHOOK: query: select * from rmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@rmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from rmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rmv_mat_view +#### A masked pattern was here #### +1 fred +PREHOOK: query: insert into rmv_table values(2, 'barney'), (11, 'betty') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@rmv_table +POSTHOOK: query: insert into rmv_table values(2, 'barney'), (11, 'betty') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@rmv_table +POSTHOOK: Lineage: rmv_table.cint EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: rmv_table.cstring1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: alter materialized view rmv_mat_view rebuild +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@rmv_table +PREHOOK: Output: database:default +PREHOOK: Output: default@rmv_mat_view +POSTHOOK: query: alter materialized view rmv_mat_view rebuild +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@rmv_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@rmv_mat_view +PREHOOK: query: select * from rmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@rmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from rmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@rmv_mat_view +#### A masked pattern was here #### +1 fred +2 barney diff --git service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java index 87ac39b..2f50b94 100644 --- service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java +++ service/src/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java @@ -37,6 +37,7 @@ public enum ClassicTableTypes { TABLE, VIEW, + MATERIALIZED_VIEW, } private final Map hiveToClientMap = new HashMap(); @@ -49,11 +50,15 @@ public ClassicTableTypeMapping () { ClassicTableTypes.TABLE.toString()); hiveToClientMap.put(TableType.VIRTUAL_VIEW.toString(), ClassicTableTypes.VIEW.toString()); + hiveToClientMap.put(TableType.MATERIALIZED_VIEW.toString(), + ClassicTableTypes.MATERIALIZED_VIEW.toString()); clientToHiveMap.put(ClassicTableTypes.TABLE.toString(), TableType.MANAGED_TABLE.toString()); clientToHiveMap.put(ClassicTableTypes.VIEW.toString(), TableType.VIRTUAL_VIEW.toString()); + clientToHiveMap.put(ClassicTableTypes.MATERIALIZED_VIEW.toString(), + TableType.MATERIALIZED_VIEW.toString()); } @Override