diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 92b4e5b..ec5d693 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -124,6 +124,8 @@ private final String stagingDir; private Heartbeater heartbeater; + + private boolean skipTableMasking; public Context(Configuration conf) throws IOException { this(conf, generateExecutionId()); @@ -778,4 +780,12 @@ public void checkHeartbeaterLockException() throws LockException { throw getHeartbeater().getLockException(); } } + + public boolean isSkipTableMasking() { + return skipTableMasking; + } + + public void setSkipTableMasking(boolean skipTableMasking) { + this.skipTableMasking = skipTableMasking; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 1e44ccf..8e5a3da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -231,6 +231,13 @@ public CalcitePlanner(QueryState queryState) throws SemanticException { } } + public void resetCalciteConfiguration() { + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)) { + runCBO = true; + disableSemJoinReordering = true; + } + } + @Override @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { @@ -2958,7 +2965,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : SemanticAnalyzer .getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), expr, col_list, - excludedColumns, inputRR, starRR, pos, out_rwsch, tabAliasesForAllProjs, true); + excludedColumns, inputRR, starRR, pos, out_rwsch, tableMask.isEnabled() ? qb.getAliases() : tabAliasesForAllProjs, true); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause @@ -2970,7 +2977,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(SemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()), null, expr, col_list, excludedColumns, inputRR, starRR, pos, out_rwsch, - tabAliasesForAllProjs, true); + tableMask.isEnabled() ? qb.getAliases() : tabAliasesForAllProjs, true); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(SemanticAnalyzer.unescapeIdentifier(expr.getChild(0) @@ -2986,7 +2993,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) SemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText()), SemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0).getText() .toLowerCase()), expr, col_list, excludedColumns, inputRR, starRR, pos, - out_rwsch, tabAliasesForAllProjs, true); + out_rwsch, tableMask.isEnabled() ? qb.getAliases() : tabAliasesForAllProjs, true); } else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregate)) { // Likely a malformed query eg, select hash(distinct c1) from t1; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java index f5a12a3..b85f980 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java @@ -25,14 +25,16 @@ String additionalTabInfo; String alias; ASTNode astNode; + boolean isView; public MaskAndFilterInfo(List colTypes, String additionalTabInfo, String alias, - ASTNode astNode) { + ASTNode astNode, boolean isView) { super(); this.colTypes = colTypes; this.additionalTabInfo = additionalTabInfo; this.alias = alias; this.astNode = astNode; + this.isView = isView; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 96ad809..a6d1fb8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -326,7 +326,7 @@ */ boolean rootTasksResolved; - private TableMask tableMask; + protected TableMask tableMask; CreateTableDesc tableDesc; @@ -3383,7 +3383,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, pos = Integer.valueOf(pos.intValue() + 1); matched++; - if (unparseTranslator.isEnabled()) { + if (unparseTranslator.isEnabled() || tableMask.isEnabled()) { if (replacementText.length() > 0) { replacementText.append(", "); } @@ -3399,6 +3399,8 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, if (unparseTranslator.isEnabled()) { unparseTranslator.addTranslation(sel, replacementText.toString()); + } else if (tableMask.isEnabled()) { + tableMask.addTranslation(sel, replacementText.toString()); } return pos; } @@ -4109,6 +4111,7 @@ static boolean isRegex(String pattern, HiveConf conf) { ASTNode[] exprs = new ASTNode[exprList.getChildCount()]; String[][] aliases = new String[exprList.getChildCount()][]; boolean[] hasAsClauses = new boolean[exprList.getChildCount()]; + int offset = 0; // Iterate over all expression (either after SELECT, or in SELECT TRANSFORM) for (int i = startPosn; i < exprList.getChildCount(); ++i) { @@ -4140,7 +4143,7 @@ static boolean isRegex(String pattern, HiveConf conf) { // Get rid of TOK_SELEXPR expr = (ASTNode) child.getChild(0); String[] colRef = getColAlias(child, autogenColAliasPrfxLbl, inputRR, - autogenColAliasPrfxIncludeFuncName, i); + autogenColAliasPrfxIncludeFuncName, i + offset); tabAlias = colRef[0]; colAlias = colRef[1]; if (hasAsClause) { @@ -4152,20 +4155,16 @@ static boolean isRegex(String pattern, HiveConf conf) { aliases[i] = new String[] {tabAlias, colAlias}; hasAsClauses[i] = hasAsClause; colAliases.add(colAlias); - } - // Iterate over all expression (either after SELECT, or in SELECT TRANSFORM) - for (int i = startPosn; i < exprList.getChildCount(); ++i) { // The real expression - ASTNode expr = exprs[i]; - String tabAlias = aliases[i][0]; - String colAlias = aliases[i][1]; - boolean hasAsClause = hasAsClauses[i]; - if (expr.getType() == HiveParser.TOK_ALLCOLREF) { + int initPos = pos; pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + if (unparseTranslator.isEnabled()) { + offset += pos - initPos - 1; + } selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -10507,6 +10506,7 @@ void setCTASToken(ASTNode child) { void setInsertToken(ASTNode ast, boolean isTmpFileDest) { } + } private Table getTableObjectByName(String tableName) throws HiveException { @@ -10577,7 +10577,7 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) } basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), - new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode)); + new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView())); } if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { @@ -10590,11 +10590,13 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) List needRewritePrivObjs = tableMask .applyRowFilterAndColumnMasking(basicPrivObjs); if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) { - tableMask.setNeedsRewrite(true); for (HivePrivilegeObject privObj : needRewritePrivObjs) { MaskAndFilterInfo info = basicInfos.get(privObj); String replacementText = tableMask.create(privObj, info); - tableMask.addTableMasking(info.astNode, replacementText); + if (replacementText != null) { + tableMask.setNeedsRewrite(true); + tableMask.addTranslation(info.astNode, replacementText); + } } } } @@ -10641,7 +10643,7 @@ public ASTNode rewriteASTWithMaskAndFilter(ASTNode ast) throws SemanticException } // 2. rewrite the AST, replace TABREF with masking/filtering if (tableMask.needsRewrite()) { - tableMask.applyTableMasking(ctx.getTokenRewriteStream()); + tableMask.applyTranslations(ctx.getTokenRewriteStream()); String rewrittenQuery = ctx.getTokenRewriteStream().toString(ast.getTokenStartIndex(), ast.getTokenStopIndex()); ASTNode rewrittenTree; @@ -10716,13 +10718,10 @@ else if(ast.getChild(0).getType() == HiveParser.TOK_FALSE) { return false; } - // masking and filtering should be done here + // masking and filtering should be created here // the basic idea is similar to unparseTranslator. - tableMask = new TableMask(this, conf); - if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) { - child = rewriteASTWithMaskAndFilter(ast); - } - + tableMask = new TableMask(this, conf, ctx); + // 4. continue analyzing from the child ASTNode. Phase1Ctx ctx_1 = initPhase1Ctx(); preProcessForInsert(child, qb); @@ -10792,6 +10791,20 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce // 2. Gen OP Tree from resolved Parse Tree Operator sinkOp = genOPTree(ast, plannerCtx); + if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) { + // Here we rewrite the * and also the masking table + ASTNode tree = rewriteASTWithMaskAndFilter(ast); + if (tree != ast) { + ctx.setSkipTableMasking(true); + init(true); + genResolvedParseTree(tree, plannerCtx); + if (this instanceof CalcitePlanner) { + ((CalcitePlanner) this).resetCalciteConfiguration(); + } + sinkOp = genOPTree(tree, plannerCtx); + } + } + // 3. Deduce Resultset Schema if (createVwDesc != null) { resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index 1686f36..e34351c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -17,16 +17,16 @@ */ package org.apache.hadoop.hive.ql.parse; -import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.antlr.runtime.TokenRewriteStream; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; @@ -46,10 +46,12 @@ private boolean enable; private boolean needsRewrite; private HiveAuthzContext queryContext; + private HiveConf conf; - public TableMask(SemanticAnalyzer analyzer, HiveConf conf) throws SemanticException { + public TableMask(SemanticAnalyzer analyzer, HiveConf conf, Context ctx) throws SemanticException { try { authorizer = SessionState.get().getAuthorizerV2(); + this.conf = conf; String cmdString = analyzer.ctx.getCmd(); SessionState ss = SessionState.get(); HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder(); @@ -57,7 +59,7 @@ public TableMask(SemanticAnalyzer analyzer, HiveConf conf) throws SemanticExcept ctxBuilder.setUserIpAddress(ss.getUserIpAddress()); ctxBuilder.setForwardedAddresses(ss.getForwardedAddresses()); queryContext = ctxBuilder.build(); - if (authorizer != null && needTransform()) { + if (authorizer != null && needTransform() && !ctx.isSkipTableMasking()) { enable = true; translator = new UnparseTranslator(conf); translator.enable(); @@ -83,6 +85,8 @@ public boolean needTransform() throws SemanticException { public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) throws SemanticException { + boolean doColumnMasking = false; + boolean doRowFiltering = false; StringBuilder sb = new StringBuilder(); sb.append("(SELECT "); boolean firstOne = true; @@ -107,38 +111,54 @@ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFi String colName = privObject.getColumns().get(index); if (!expr.equals(colName)) { // CAST(expr AS COLTYPE) AS COLNAME - sb.append("CAST(" + expr + " AS " + colTypes.get(index) + ") AS `" + colName + "`"); + sb.append("CAST(" + expr + " AS " + colTypes.get(index) + ") AS " + + HiveUtils.unparseIdentifier(colName, conf)); + doColumnMasking = true; } else { - sb.append(expr); + sb.append(HiveUtils.unparseIdentifier(colName, conf)); } } - } else { - for (int index = 0; index < privObject.getColumns().size(); index++) { - String expr = privObject.getColumns().get(index); - if (!firstOne) { - sb.append(", "); - } else { - firstOne = false; - } - sb.append(expr); + } + if (!doColumnMasking) { + sb = new StringBuilder(); + sb.append("(SELECT *"); + } + + if (!maskAndFilterInfo.isView) { + // put all virtual columns in RowResolver. + Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); + while (vcs.hasNext()) { + VirtualColumn vc = vcs.next(); + sb.append(", " + vc.getName()); } } - sb.append(" FROM `" + privObject.getDbname() + "`.`" + privObject.getObjectName() + "`"); + + sb.append(" FROM "); + sb.append(HiveUtils.unparseIdentifier(privObject.getDbname(), conf)); + sb.append("."); + sb.append(HiveUtils.unparseIdentifier(privObject.getObjectName(), conf)); sb.append(" " + maskAndFilterInfo.additionalTabInfo); String filter = privObject.getRowFilterExpression(); if (filter != null) { sb.append(" WHERE " + filter); + doRowFiltering = true; + } + sb.append(")" + HiveUtils.unparseIdentifier(maskAndFilterInfo.alias, conf)); + + if (!doColumnMasking && !doRowFiltering) { + // nothing to do + return null; + } else { + LOG.debug("TableMask creates `" + sb.toString() + "`"); + return sb.toString(); } - sb.append(")" + maskAndFilterInfo.alias); - LOG.debug("TableMask creates `" + sb.toString() + "`"); - return sb.toString(); } - void addTableMasking(ASTNode node, String replacementText) throws SemanticException { + void addTranslation(ASTNode node, String replacementText) throws SemanticException { translator.addTranslation(node, replacementText); } - void applyTableMasking(TokenRewriteStream tokenRewriteStream) throws SemanticException { + void applyTranslations(TokenRewriteStream tokenRewriteStream) throws SemanticException { translator.applyTranslations(tokenRewriteStream); } diff --git a/ql/src/test/queries/clientpositive/masking_6.q b/ql/src/test/queries/clientpositive/masking_6.q new file mode 100644 index 0000000..e266741 --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_6.q @@ -0,0 +1,28 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, value, '12' from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; diff --git a/ql/src/test/queries/clientpositive/masking_7.q b/ql/src/test/queries/clientpositive/masking_7.q new file mode 100644 index 0000000..6d4784d --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_7.q @@ -0,0 +1,28 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; diff --git a/ql/src/test/queries/clientpositive/masking_8.q b/ql/src/test/queries/clientpositive/masking_8.q new file mode 100644 index 0000000..7dfeda8 --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_8.q @@ -0,0 +1,38 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop table masking_test; + +create table masking_test as select cast(key as int) as key, value, '12' from src; + +explain select *, ROW__ID from masking_test; + +select *, ROW__ID from masking_test; + +explain select * from masking_test; + +select * from masking_test; + +explain select INPUT__FILE__NAME, *, ROW__ID from masking_test; + +select INPUT__FILE__NAME, *, ROW__ID from masking_test; + +drop table masking_test; + +create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select ROW__ID, * from masking_test; + +select ROW__ID, * from masking_test; + +drop table masking_test; + +create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', INPUT__FILE__NAME, '12', '12', '12', '12', '12' + from src; + +explain select INPUT__FILE__NAME, *, ROW__ID from masking_test; + +select INPUT__FILE__NAME, *, ROW__ID from masking_test; diff --git a/ql/src/test/queries/clientpositive/masking_9.q b/ql/src/test/queries/clientpositive/masking_9.q new file mode 100644 index 0000000..c8b54ce --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_9.q @@ -0,0 +1,9 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop table masking_test; + +create temporary table masking_test as select cast(key as int) as key, value from src; + +explain select ROW__ID from masking_test; +select ROW__ID from masking_test; diff --git a/ql/src/test/queries/clientpositive/view_alias.q b/ql/src/test/queries/clientpositive/view_alias.q new file mode 100644 index 0000000..e85ba12 --- /dev/null +++ b/ql/src/test/queries/clientpositive/view_alias.q @@ -0,0 +1,34 @@ +drop view v; +create view v as select key, '12' from src; +desc formatted v; +select * from v order by `_c1` limit 5; + +drop view v; +create view v as select key as `_c1`, '12' from src; +desc formatted v; +select * from v order by `_c1` limit 5; + +drop view v; +create view v as select *, '12' from src; +desc formatted v; +select * from v order by `_c2` limit 5; + +drop view v; +create view v as select *, '12' as `_c121` from src; +desc formatted v; +select * from v order by `_c121` limit 5; + +drop view v; +create view v as select key, count(*) from src group by key; +desc formatted v; +select * from v order by `_c1` limit 5; + + +drop view v; +create table a (ca string, caa string); +create table b (cb string, cbb string); +insert into a select * from src limit 5; +insert into b select * from src limit 5; +create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb; +desc formatted v; +select * from v order by `_c3` limit 5; diff --git a/ql/src/test/results/clientpositive/masking_6.q.out b/ql/src/test/results/clientpositive/masking_6.q.out new file mode 100644 index 0000000..fb8c90f --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_6.q.out @@ -0,0 +1,238 @@ +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view masking_test as select cast(key as int) as key, value, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, value, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0_lav 12 +4 4_lav 12 +8 8_lav 12 +0 0_lav 12 +0 0_lav 12 +2 2_lav 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 4_lav 12 +8 8_lav 12 +2 2_lav 12 +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 12 12 12 12 12 12 12 12 12 12 12 +4 12 12 12 12 12 12 12 12 12 12 12 +8 12 12 12 12 12 12 12 12 12 12 12 +0 12 12 12 12 12 12 12 12 12 12 12 +0 12 12 12 12 12 12 12 12 12 12 12 +2 12 12 12 12 12 12 12 12 12 12 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 12 12 12 12 12 12 12 12 12 12 12 +8 12 12 12 12 12 12 12 12 12 12 12 +2 12 12 12 12 12 12 12 12 12 12 12 diff --git a/ql/src/test/results/clientpositive/masking_7.q.out b/ql/src/test/results/clientpositive/masking_7.q.out new file mode 100644 index 0000000..1fa9311 --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_7.q.out @@ -0,0 +1,238 @@ +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0_lav 12 NULL +4 4_lav 12 NULL +8 8_lav 12 NULL +0 0_lav 12 NULL +0 0_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 4_lav 12 NULL +8 8_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), ROW__ID (type: struct), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +4 12 NULL 12 12 12 12 12 12 12 12 12 12 +8 12 NULL 12 12 12 12 12 12 12 12 12 12 +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +2 12 NULL 12 12 12 12 12 12 12 12 12 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), ROW__ID (type: struct), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 12 NULL 12 12 12 12 12 12 12 12 12 12 +8 12 NULL 12 12 12 12 12 12 12 12 12 12 +2 12 NULL 12 12 12 12 12 12 12 12 12 12 diff --git a/ql/src/test/results/clientpositive/masking_8.q.out b/ql/src/test/results/clientpositive/masking_8.q.out new file mode 100644 index 0000000..be142b0 --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_8.q.out @@ -0,0 +1,322 @@ +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table masking_test as select cast(key as int) as key, value, '12' from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test as select cast(key as int) as key, value, '12' from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +POSTHOOK: Lineage: masking_test.c2 SIMPLE [] +POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select *, ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), reverse(value) (type: string), c2 (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select *, ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +0 0_lav 12 NULL +4 4_lav 12 NULL +8 8_lav 12 NULL +0 0_lav 12 NULL +0 0_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), reverse(value) (type: string), c2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +0 0_lav 12 +4 4_lav 12 +8 8_lav 12 +0 0_lav 12 +0 0_lav 12 +2 2_lav 12 +PREHOOK: query: explain select INPUT__FILE__NAME, *, ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select INPUT__FILE__NAME, *, ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: INPUT__FILE__NAME (type: string), key (type: int), reverse(value) (type: string), c2 (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select INPUT__FILE__NAME, *, ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select INPUT__FILE__NAME, *, ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +POSTHOOK: Lineage: masking_test.c1 SIMPLE [] +POSTHOOK: Lineage: masking_test.c10 SIMPLE [] +POSTHOOK: Lineage: masking_test.c2 SIMPLE [] +POSTHOOK: Lineage: masking_test.c3 SIMPLE [] +POSTHOOK: Lineage: masking_test.c4 SIMPLE [] +POSTHOOK: Lineage: masking_test.c5 SIMPLE [] +POSTHOOK: Lineage: masking_test.c6 SIMPLE [] +POSTHOOK: Lineage: masking_test.c7 SIMPLE [] +POSTHOOK: Lineage: masking_test.c8 SIMPLE [] +POSTHOOK: Lineage: masking_test.c9 SIMPLE [] +POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain select ROW__ID, * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select ROW__ID, * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 17406 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ROW__ID (type: struct), key (type: int), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string), c5 (type: string), c6 (type: string), c7 (type: string), c8 (type: string), c9 (type: string), c10 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ROW__ID, * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select ROW__ID, * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +NULL 0 1212 12 12 12 12 12 12 12 12 12 +NULL 4 1212 12 12 12 12 12 12 12 12 12 +NULL 8 1212 12 12 12 12 12 12 12 12 12 +NULL 0 1212 12 12 12 12 12 12 12 12 12 +NULL 0 1212 12 12 12 12 12 12 12 12 12 +NULL 2 1212 12 12 12 12 12 12 12 12 12 +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', INPUT__FILE__NAME, '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', INPUT__FILE__NAME, '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +POSTHOOK: Lineage: masking_test.c1 SIMPLE [] +POSTHOOK: Lineage: masking_test.c10 SIMPLE [] +POSTHOOK: Lineage: masking_test.c11 SIMPLE [] +POSTHOOK: Lineage: masking_test.c2 SIMPLE [] +POSTHOOK: Lineage: masking_test.c3 SIMPLE [] +POSTHOOK: Lineage: masking_test.c4 SIMPLE [] +POSTHOOK: Lineage: masking_test.c5 SIMPLE [] +POSTHOOK: Lineage: masking_test.c7 SIMPLE [] +POSTHOOK: Lineage: masking_test.c8 SIMPLE [] +POSTHOOK: Lineage: masking_test.c9 SIMPLE [] +POSTHOOK: Lineage: masking_test.input__file__name SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ] +POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain select INPUT__FILE__NAME, *, ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select INPUT__FILE__NAME, *, ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 57406 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 9529 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: INPUT__FILE__NAME (type: string), key (type: int), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string), c5 (type: string), c7 (type: string), c8 (type: string), c9 (type: string), c10 (type: string), c11 (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 83 Data size: 9529 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 9529 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select INPUT__FILE__NAME, *, ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select INPUT__FILE__NAME, *, ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/masking_9.q.out b/ql/src/test/results/clientpositive/masking_9.q.out new file mode 100644 index 0000000..c337cbc --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_9.q.out @@ -0,0 +1,64 @@ +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +PREHOOK: query: create temporary table masking_test as select cast(key as int) as key, value from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create temporary table masking_test as select cast(key as int) as key, value from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +NULL +NULL +NULL +NULL +NULL +NULL diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out index 6717527..1dc2461 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out @@ -248,15 +248,19 @@ STAGE PLANS: keys: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out index 6aaab20..83cb6ca 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out @@ -2403,15 +2403,19 @@ STAGE PLANS: keys: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/view_alias.q.out b/ql/src/test/results/clientpositive/view_alias.q.out new file mode 100644 index 0000000..78ff5e2 --- /dev/null +++ b/ql/src/test/results/clientpositive/view_alias.q.out @@ -0,0 +1,427 @@ +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select key, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key, '12' from src +View Expanded Text: select `src`.`key`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 12 +27 12 +311 12 +97 12 +238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select key as `_c1`, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key as `_c1`, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key as _c1, '12' from src +View Expanded Text: select `src`.`key` as `_c1`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 12 +27 12 +311 12 +97 12 +238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select *, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select *, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string +_c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select *, '12' from src +View Expanded Text: select `src`.`key`, `src`.`value`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c2` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c2` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 val_165 12 +27 val_27 12 +311 val_311 12 +97 val_97 12 +238 val_238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select *, '12' as `_c121` from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select *, '12' as `_c121` from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string +_c121 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select *, '12' as _c121 from src +View Expanded Text: select `src`.`key`, `src`.`value`, '12' as `_c121` from `default`.`src` +PREHOOK: query: select * from v order by `_c121` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c121` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 val_165 12 +27 val_27 12 +311 val_311 12 +97 val_97 12 +238 val_238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select key, count(*) from src group by key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key, count(*) from src group by key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key, count(*) from src group by key +View Expanded Text: select `src`.`key`, count(*) from `default`.`src` group by `src`.`key` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +11 1 +105 1 +114 1 +96 1 +10 1 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create table a (ca string, caa string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a (ca string, caa string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b (cb string, cbb string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b (cb string, cbb string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: insert into a select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +POSTHOOK: query: insert into a select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Lineage: a.ca SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.caa SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into b select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@b +POSTHOOK: query: insert into b select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@b +POSTHOOK: Lineage: b.cb SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.cbb SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@a +PREHOOK: Input: default@b +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@a +POSTHOOK: Input: default@b +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +_c0 string +ca string +caa string +_c3 int +cb string +cbb string +_c6 int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +View Expanded Text: select '010', `a`.`ca`, `a`.`caa`, 121, `b`.`cb`, `b`.`cbb`, 234 from `default`.`a` join `default`.`b` on `a`.`ca` = `b`.`cb` +PREHOOK: query: select * from v order by `_c3` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@a +PREHOOK: Input: default@b +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c3` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@a +POSTHOOK: Input: default@b +POSTHOOK: Input: default@v +#### A masked pattern was here #### +010 86 val_86 121 86 val_86 234 +010 311 val_311 121 311 val_311 234 +010 27 val_27 121 27 val_27 234 +010 238 val_238 121 238 val_238 234 +010 165 val_165 121 165 val_165 234