diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 37bc153631..0d74671160 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -386,6 +386,9 @@ MASKING_FILTERING_ON_ACID_NOT_SUPPORTED(10287, "Detected {0}.{1} has row masking/column filtering enabled, " + "which is not supported for query involving ACID operations", true), + MASKING_FILTERING_ON_MATERIALIZED_VIEWS_SOURCES(10288, + "Querying directly materialized view contents is not supported since we detected {0}.{1} " + + "used by materialized view has row masking/column filtering enabled", true), UPDATEDELETE_PARSE_ERROR(10290, "Encountered parse error while parsing rewritten merge/update or " + "delete query"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 474c793ec3..463880587e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11930,30 +11930,36 @@ private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set colNames = new ArrayList<>(); - List colTypes = new ArrayList<>(); - for (FieldSchema col : table.getAllCols()) { - colNames.add(col.getName()); - colTypes.add(col.getType()); - } + if (table.isMaterializedView()) { + // When we are querying a materialized view directly, we check whether the source tables + // do not apply any policies. + for (String qName : table.getCreationMetadata().getTablesUsed()) { + table = getTable(db, qName, tabNameToTabObject); + if (table == null) { + // This should not happen + throw new SemanticException("Table " + qName + " not found when trying to obtain it to check masking/filtering " + + "policies"); + } - basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), - new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView(), table.isNonNative())); + List colNames = new ArrayList<>(); + extractColumnInfos(table, colNames, new ArrayList<>()); + + basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), null); + } + } else { + List colNames = new ArrayList<>(); + List colTypes = new ArrayList<>(); + extractColumnInfos(table, colNames, colTypes); + + basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), + new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView(), table.isNonNative())); + } } if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { @@ -11968,20 +11974,52 @@ private static void walkASTMarkTABREF(TableMask tableMask, ASTNode ast, Set tabNameToTabObject) { + Table table = null; + try { + if (!tabNameToTabObject.containsKey(tabIdName)) { + table = db.getTable(tabIdName, true); + tabNameToTabObject.put(tabIdName, table); + } else { + table = tabNameToTabObject.get(tabIdName); + } + } catch (HiveException e) { + // Table may not be found when materialization of CTE is on. + STATIC_LOG.debug("Table " + tabIdName + " is not found in walkASTMarkTABREF."); + } + return table; + } + + private static void extractColumnInfos(Table table, List colNames, List colTypes) { + for (FieldSchema col : table.getAllCols()) { + colNames.add(col.getName()); + colTypes.add(col.getType()); + } + } + // We walk through the AST. // We replace all the TOK_TABREF by adding additional masking and filter if // the table needs to be masked or filtered. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index ee93cf65fd..b1db5477cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -83,26 +83,43 @@ public boolean needTransform() throws SemanticException { return authorizer.needTransform(); } - public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) + public boolean needsMaskingOrFiltering(HivePrivilegeObject privObject) throws SemanticException { - boolean doColumnMasking = false; - boolean doRowFiltering = false; - StringBuilder sb = new StringBuilder(); - sb.append("(SELECT "); - boolean firstOne = true; + String filter = privObject.getRowFilterExpression(); + if (filter != null) { + return true; + } List exprs = privObject.getCellValueTransformers(); if (exprs != null) { if (exprs.size() != privObject.getColumns().size()) { throw new SemanticException("Expect " + privObject.getColumns().size() + " columns in " + privObject.getObjectName() + ", but only find " + exprs.size()); } - List colTypes = maskAndFilterInfo.colTypes; for (int index = 0; index < exprs.size(); index++) { String expr = exprs.get(index); if (expr == null) { throw new SemanticException("Expect string type CellValueTransformer in " + privObject.getObjectName() + ", but only find null"); } + String colName = privObject.getColumns().get(index); + if (!expr.equals(colName)) { + return true; + } + } + } + return false; + } + + public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) { + boolean doColumnMasking = false; + StringBuilder sb = new StringBuilder(); + sb.append("(SELECT "); + boolean firstOne = true; + List exprs = privObject.getCellValueTransformers(); + if (exprs != null) { + List colTypes = maskAndFilterInfo.colTypes; + for (int index = 0; index < exprs.size(); index++) { + String expr = exprs.get(index); if (!firstOne) { sb.append(", "); } else { @@ -141,17 +158,11 @@ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFi String filter = privObject.getRowFilterExpression(); if (filter != null) { sb.append(" WHERE " + filter); - doRowFiltering = true; } sb.append(")" + HiveUtils.unparseIdentifier(maskAndFilterInfo.alias, conf)); - if (!doColumnMasking && !doRowFiltering) { - // nothing to do - return null; - } else { - LOG.debug("TableMask creates `" + sb.toString() + "`"); - return sb.toString(); - } + LOG.debug("TableMask creates `" + sb.toString() + "`"); + return sb.toString(); } void addTranslation(ASTNode node, String replacementText) throws SemanticException { diff --git a/ql/src/test/queries/clientnegative/masking_mv.q b/ql/src/test/queries/clientnegative/masking_mv.q new file mode 100644 index 0000000000..deb5c4a80f --- /dev/null +++ b/ql/src/test/queries/clientnegative/masking_mv.q @@ -0,0 +1,17 @@ +--! qt:dataset:srcpart +--! qt:dataset:src +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src; + +explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv`; +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv`; + +select key from `masking_test_view_n_mv`; diff --git a/ql/src/test/queries/clientpositive/masking_mv.q b/ql/src/test/queries/clientpositive/masking_mv.q new file mode 100644 index 0000000000..51da471258 --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_mv.q @@ -0,0 +1,43 @@ +--! qt:dataset:srcpart +--! qt:dataset:src +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src; + +explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv`; +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv`; +describe formatted `masking_test_view_n_mv`; + +-- This will use the MV and will apply masking/filtering on top +-- of the source table, not any policy that applies on the MV +explain +select key from `masking_test_n_mv`; +select key from `masking_test_n_mv`; + +create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src; + +explain +create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx`; +create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx`; +describe formatted `masking_test_view_n_mv_2`; + +-- This is allowed because source tables do not use a masking +-- or filtering. Policy is not applied on MV as expected +explain +select key from `masking_test_view_n_mv_2` order by key; +select key from `masking_test_view_n_mv_2` order by key; + +drop materialized view `masking_test_view_n_mv`; +drop materialized view `masking_test_view_n_mv_2`; +drop table `masking_test_n_mv`; +drop table `srcTnx`; diff --git a/ql/src/test/results/clientnegative/masking_mv.q.out b/ql/src/test/results/clientnegative/masking_mv.q.out new file mode 100644 index 0000000000..1c40f50d41 --- /dev/null +++ b/ql/src/test/results/clientnegative/masking_mv.q.out @@ -0,0 +1,119 @@ +PREHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_n_mv +POSTHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_n_mv +POSTHOOK: Lineage: masking_test_n_mv.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: masking_test_n_mv.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-8 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-8 + Stage-9 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test_n_mv + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_view_n_mv + Execution mode: vectorized + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Create View Operator: + Create View + columns: key int + expanded text: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv` + name: default.masking_test_view_n_mv + original text: select key from masking_test_n_mv + rewrite enabled: true + + Stage: Stage-2 + Stats Work + Basic Stats Work: + + Stage: Stage-9 + Materialized View Work + + Stage: Stage-3 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-5 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@masking_test_n_mv +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_view_n_mv +POSTHOOK: query: create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@masking_test_n_mv +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_view_n_mv +FAILED: SemanticException [Error 10288]: Querying directly materialized view contents is not supported since we detected default.masking_test_n_mv used by materialized view has row masking/column filtering enabled diff --git a/ql/src/test/results/clientpositive/masking_mv.q.out b/ql/src/test/results/clientpositive/masking_mv.q.out new file mode 100644 index 0000000000..468c032e4d --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_mv.q.out @@ -0,0 +1,953 @@ +PREHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_n_mv +POSTHOOK: query: create table `masking_test_n_mv` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_n_mv +POSTHOOK: Lineage: masking_test_n_mv.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: masking_test_n_mv.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: explain +create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-8 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-8 + Stage-9 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test_n_mv + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_view_n_mv + Execution mode: vectorized + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Create View Operator: + Create View + columns: key int + expanded text: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv` + name: default.masking_test_view_n_mv + original text: select key from masking_test_n_mv + rewrite enabled: true + + Stage: Stage-2 + Stats Work + Basic Stats Work: + + Stage: Stage-9 + Materialized View Work + + Stage: Stage-3 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-5 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@masking_test_n_mv +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_view_n_mv +POSTHOOK: query: create materialized view `masking_test_view_n_mv` as +select key from `masking_test_n_mv` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@masking_test_n_mv +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_view_n_mv +PREHOOK: query: describe formatted `masking_test_view_n_mv` +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@masking_test_view_n_mv +POSTHOOK: query: describe formatted `masking_test_view_n_mv` +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@masking_test_view_n_mv +# col_name data_type comment +key int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 1 + numRows 500 + rawDataSize 2000 + totalSize 974 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: select key from masking_test_n_mv +Expanded Query: select `masking_test_n_mv`.`key` from `default`.`masking_test_n_mv` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: explain +select key from `masking_test_n_mv` +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from `masking_test_n_mv` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.masking_test_view_n_mv + filterExpr: ((key < 10) and (0 = (key % 2))) (type: boolean) + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((0 = (key % 2)) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 332 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key from `masking_test_n_mv` +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_n_mv +PREHOOK: Input: default@masking_test_view_n_mv +#### A masked pattern was here #### +POSTHOOK: query: select key from `masking_test_n_mv` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_n_mv +POSTHOOK: Input: default@masking_test_view_n_mv +#### A masked pattern was here #### +0 +4 +8 +0 +0 +2 +PREHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@srcTnx +POSTHOOK: query: create table `srcTnx` stored as orc TBLPROPERTIES ('transactional'='true') as +select cast(key as int) as key, value from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcTnx +POSTHOOK: Lineage: srctnx.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srctnx.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: explain +create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-8 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-8 + Stage-9 depends on stages: Stage-2 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: srctnx + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 30200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.masking_test_view_n_mv_2 + Execution mode: vectorized + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Create View Operator: + Create View + columns: key int + expanded text: select `srctnx`.`key` from `default`.`srcTnx` + name: default.masking_test_view_n_mv_2 + original text: select key from srcTnx + rewrite enabled: true + + Stage: Stage-2 + Stats Work + Basic Stats Work: + + Stage: Stage-9 + Materialized View Work + + Stage: Stage-3 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-5 + Merge File Operator + Map Operator Tree: + ORC File Merge Operator + merge level: stripe + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx` +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@srctnx +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test_view_n_mv_2 +POSTHOOK: query: create materialized view `masking_test_view_n_mv_2` as +select key from `srcTnx` +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@srctnx +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test_view_n_mv_2 +PREHOOK: query: describe formatted `masking_test_view_n_mv_2` +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@masking_test_view_n_mv_2 +POSTHOOK: query: describe formatted `masking_test_view_n_mv_2` +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@masking_test_view_n_mv_2 +# col_name data_type comment +key int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 1 + numRows 500 + rawDataSize 2000 + totalSize 974 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: select key from srcTnx +Expanded Query: select `srctnx`.`key` from `default`.`srcTnx` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: explain +select key from `masking_test_view_n_mv_2` order by key +PREHOOK: type: QUERY +POSTHOOK: query: explain +select key from `masking_test_view_n_mv_2` order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test_view_n_mv_2 + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key from `masking_test_view_n_mv_2` order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test_view_n_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: select key from `masking_test_view_n_mv_2` order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test_view_n_mv_2 +#### A masked pattern was here #### +0 +0 +0 +2 +4 +5 +5 +5 +8 +9 +10 +11 +12 +12 +15 +15 +17 +18 +18 +19 +20 +24 +24 +26 +26 +27 +28 +30 +33 +34 +35 +35 +35 +37 +37 +41 +42 +42 +43 +44 +47 +51 +51 +53 +54 +57 +58 +58 +64 +65 +66 +67 +67 +69 +70 +70 +70 +72 +72 +74 +76 +76 +77 +78 +80 +82 +83 +83 +84 +84 +85 +86 +87 +90 +90 +90 +92 +95 +95 +96 +97 +97 +98 +98 +100 +100 +103 +103 +104 +104 +105 +111 +113 +113 +114 +116 +118 +118 +119 +119 +119 +120 +120 +125 +125 +126 +128 +128 +128 +129 +129 +131 +133 +134 +134 +136 +137 +137 +138 +138 +138 +138 +143 +145 +146 +146 +149 +149 +150 +152 +152 +153 +155 +156 +157 +158 +160 +162 +163 +164 +164 +165 +165 +166 +167 +167 +167 +168 +169 +169 +169 +169 +170 +172 +172 +174 +174 +175 +175 +176 +176 +177 +178 +179 +179 +180 +181 +183 +186 +187 +187 +187 +189 +190 +191 +191 +192 +193 +193 +193 +194 +195 +195 +196 +197 +197 +199 +199 +199 +200 +200 +201 +202 +203 +203 +205 +205 +207 +207 +208 +208 +208 +209 +209 +213 +213 +214 +216 +216 +217 +217 +218 +219 +219 +221 +221 +222 +223 +223 +224 +224 +226 +228 +229 +229 +230 +230 +230 +230 +230 +233 +233 +235 +237 +237 +238 +238 +239 +239 +241 +242 +242 +244 +247 +248 +249 +252 +255 +255 +256 +256 +257 +258 +260 +262 +263 +265 +265 +266 +272 +272 +273 +273 +273 +274 +275 +277 +277 +277 +277 +278 +278 +280 +280 +281 +281 +282 +282 +283 +284 +285 +286 +287 +288 +288 +289 +291 +292 +296 +298 +298 +298 +302 +305 +306 +307 +307 +308 +309 +309 +310 +311 +311 +311 +315 +316 +316 +316 +317 +317 +318 +318 +318 +321 +321 +322 +322 +323 +325 +325 +327 +327 +327 +331 +331 +332 +333 +333 +335 +336 +338 +339 +341 +342 +342 +344 +344 +345 +348 +348 +348 +348 +348 +351 +353 +353 +356 +360 +362 +364 +365 +366 +367 +367 +368 +369 +369 +369 +373 +374 +375 +377 +378 +379 +382 +382 +384 +384 +384 +386 +389 +392 +393 +394 +395 +395 +396 +396 +396 +397 +397 +399 +399 +400 +401 +401 +401 +401 +401 +402 +403 +403 +403 +404 +404 +406 +406 +406 +406 +407 +409 +409 +409 +411 +413 +413 +414 +414 +417 +417 +417 +418 +419 +421 +424 +424 +427 +429 +429 +430 +430 +430 +431 +431 +431 +432 +435 +436 +437 +438 +438 +438 +439 +439 +443 +444 +446 +448 +449 +452 +453 +454 +454 +454 +455 +457 +458 +458 +459 +459 +460 +462 +462 +463 +463 +466 +466 +466 +467 +468 +468 +468 +468 +469 +469 +469 +469 +469 +470 +472 +475 +477 +478 +478 +479 +480 +480 +480 +481 +482 +483 +484 +485 +487 +489 +489 +489 +489 +490 +491 +492 +492 +493 +494 +495 +496 +497 +498 +498 +498 +PREHOOK: query: drop materialized view `masking_test_view_n_mv` +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@masking_test_view_n_mv +PREHOOK: Output: default@masking_test_view_n_mv +POSTHOOK: query: drop materialized view `masking_test_view_n_mv` +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@masking_test_view_n_mv +POSTHOOK: Output: default@masking_test_view_n_mv +PREHOOK: query: drop materialized view `masking_test_view_n_mv_2` +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@masking_test_view_n_mv_2 +PREHOOK: Output: default@masking_test_view_n_mv_2 +POSTHOOK: query: drop materialized view `masking_test_view_n_mv_2` +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@masking_test_view_n_mv_2 +POSTHOOK: Output: default@masking_test_view_n_mv_2 +PREHOOK: query: drop table `masking_test_n_mv` +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@masking_test_n_mv +PREHOOK: Output: default@masking_test_n_mv +POSTHOOK: query: drop table `masking_test_n_mv` +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@masking_test_n_mv +POSTHOOK: Output: default@masking_test_n_mv +PREHOOK: query: drop table `srcTnx` +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srctnx +PREHOOK: Output: default@srctnx +POSTHOOK: query: drop table `srcTnx` +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srctnx +POSTHOOK: Output: default@srctnx