diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 1e44ccf..1cee311 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2958,7 +2958,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) if (expr.getType() == HiveParser.TOK_ALLCOLREF) { pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : SemanticAnalyzer .getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), expr, col_list, - excludedColumns, inputRR, starRR, pos, out_rwsch, tabAliasesForAllProjs, true); + excludedColumns, inputRR, starRR, pos, out_rwsch, qb.getAliases(), true); selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause @@ -2970,7 +2970,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) // We don't allow this for ExprResolver - the Group By case pos = genColListRegex(SemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()), null, expr, col_list, excludedColumns, inputRR, starRR, pos, out_rwsch, - tabAliasesForAllProjs, true); + qb.getAliases(), true); } else if (expr.getType() == HiveParser.DOT && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL && inputRR.hasTableAlias(SemanticAnalyzer.unescapeIdentifier(expr.getChild(0) @@ -2986,7 +2986,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) SemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText()), SemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0).getText() .toLowerCase()), expr, col_list, excludedColumns, inputRR, starRR, pos, - out_rwsch, tabAliasesForAllProjs, true); + out_rwsch, qb.getAliases(), true); } else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregate)) { // Likely a malformed query eg, select hash(distinct c1) from t1; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java index f5a12a3..b85f980 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaskAndFilterInfo.java @@ -25,14 +25,16 @@ String additionalTabInfo; String alias; ASTNode astNode; + boolean isView; public MaskAndFilterInfo(List colTypes, String additionalTabInfo, String alias, - ASTNode astNode) { + ASTNode astNode, boolean isView) { super(); this.colTypes = colTypes; this.additionalTabInfo = additionalTabInfo; this.alias = alias; this.astNode = astNode; + this.isView = isView; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 96ad809..39735ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3383,7 +3383,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, pos = Integer.valueOf(pos.intValue() + 1); matched++; - if (unparseTranslator.isEnabled()) { + if (unparseTranslator.isEnabled() || tableMask.isEnabled()) { if (replacementText.length() > 0) { replacementText.append(", "); } @@ -3399,6 +3399,8 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, if (unparseTranslator.isEnabled()) { unparseTranslator.addTranslation(sel, replacementText.toString()); + } else if (tableMask.isEnabled()) { + tableMask.addTranslation(sel, replacementText.toString()); } return pos; } @@ -4109,6 +4111,7 @@ static boolean isRegex(String pattern, HiveConf conf) { ASTNode[] exprs = new ASTNode[exprList.getChildCount()]; String[][] aliases = new String[exprList.getChildCount()][]; boolean[] hasAsClauses = new boolean[exprList.getChildCount()]; + int offset = 0; // Iterate over all expression (either after SELECT, or in SELECT TRANSFORM) for (int i = startPosn; i < exprList.getChildCount(); ++i) { @@ -4140,7 +4143,7 @@ static boolean isRegex(String pattern, HiveConf conf) { // Get rid of TOK_SELEXPR expr = (ASTNode) child.getChild(0); String[] colRef = getColAlias(child, autogenColAliasPrfxLbl, inputRR, - autogenColAliasPrfxIncludeFuncName, i); + autogenColAliasPrfxIncludeFuncName, i + offset); tabAlias = colRef[0]; colAlias = colRef[1]; if (hasAsClause) { @@ -4152,20 +4155,16 @@ static boolean isRegex(String pattern, HiveConf conf) { aliases[i] = new String[] {tabAlias, colAlias}; hasAsClauses[i] = hasAsClause; colAliases.add(colAlias); - } - // Iterate over all expression (either after SELECT, or in SELECT TRANSFORM) - for (int i = startPosn; i < exprList.getChildCount(); ++i) { // The real expression - ASTNode expr = exprs[i]; - String tabAlias = aliases[i][0]; - String colAlias = aliases[i][1]; - boolean hasAsClause = hasAsClauses[i]; - if (expr.getType() == HiveParser.TOK_ALLCOLREF) { + int initPos = pos; pos = genColListRegex(".*", expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + if (unparseTranslator.isEnabled()) { + offset += pos - initPos - 1; + } selectStar = true; } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause && !inputRR.getIsExprResolver() @@ -10507,6 +10506,7 @@ void setCTASToken(ASTNode child) { void setInsertToken(ASTNode ast, boolean isTmpFileDest) { } + } private Table getTableObjectByName(String tableName) throws HiveException { @@ -10577,7 +10577,7 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) } basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), - new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode)); + new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView())); } if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) { for (Node child : astNode.getChildren()) { @@ -10594,7 +10594,7 @@ private void walkASTMarkTABREF(ASTNode ast, Set cteAlias) for (HivePrivilegeObject privObj : needRewritePrivObjs) { MaskAndFilterInfo info = basicInfos.get(privObj); String replacementText = tableMask.create(privObj, info); - tableMask.addTableMasking(info.astNode, replacementText); + tableMask.addTranslation(info.astNode, replacementText); } } } @@ -10641,7 +10641,7 @@ public ASTNode rewriteASTWithMaskAndFilter(ASTNode ast) throws SemanticException } // 2. rewrite the AST, replace TABREF with masking/filtering if (tableMask.needsRewrite()) { - tableMask.applyTableMasking(ctx.getTokenRewriteStream()); + tableMask.applyTranslations(ctx.getTokenRewriteStream()); String rewrittenQuery = ctx.getTokenRewriteStream().toString(ast.getTokenStartIndex(), ast.getTokenStopIndex()); ASTNode rewrittenTree; @@ -10716,13 +10716,10 @@ else if(ast.getChild(0).getType() == HiveParser.TOK_FALSE) { return false; } - // masking and filtering should be done here + // masking and filtering should be created here // the basic idea is similar to unparseTranslator. tableMask = new TableMask(this, conf); - if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) { - child = rewriteASTWithMaskAndFilter(ast); - } - + // 4. continue analyzing from the child ASTNode. Phase1Ctx ctx_1 = initPhase1Ctx(); preProcessForInsert(child, qb); @@ -10792,6 +10789,14 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce // 2. Gen OP Tree from resolved Parse Tree Operator sinkOp = genOPTree(ast, plannerCtx); + if (!unparseTranslator.isEnabled() && tableMask.isEnabled()) { + // need to rewrite star + ASTNode child = rewriteASTWithMaskAndFilter(ast); + init(true); + genResolvedParseTree(child, plannerCtx); + sinkOp = genOPTree(ast, plannerCtx);; + } + // 3. Deduce Resultset Schema if (createVwDesc != null) { resultSchema = convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java index 1686f36..77d590d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableMask.java @@ -17,16 +17,15 @@ */ package org.apache.hadoop.hive.ql.parse; -import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.antlr.runtime.TokenRewriteStream; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; -import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; @@ -46,10 +45,12 @@ private boolean enable; private boolean needsRewrite; private HiveAuthzContext queryContext; + private HiveConf conf; public TableMask(SemanticAnalyzer analyzer, HiveConf conf) throws SemanticException { try { authorizer = SessionState.get().getAuthorizerV2(); + this.conf = conf; String cmdString = analyzer.ctx.getCmd(); SessionState ss = SessionState.get(); HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder(); @@ -83,6 +84,8 @@ public boolean needTransform() throws SemanticException { public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFilterInfo) throws SemanticException { + boolean doColumnMasking = false; + boolean doRowFiltering = false; StringBuilder sb = new StringBuilder(); sb.append("(SELECT "); boolean firstOne = true; @@ -107,38 +110,54 @@ public String create(HivePrivilegeObject privObject, MaskAndFilterInfo maskAndFi String colName = privObject.getColumns().get(index); if (!expr.equals(colName)) { // CAST(expr AS COLTYPE) AS COLNAME - sb.append("CAST(" + expr + " AS " + colTypes.get(index) + ") AS `" + colName + "`"); + sb.append("CAST(" + expr + " AS " + colTypes.get(index) + ") AS " + + HiveUtils.unparseIdentifier(colName, conf)); + doColumnMasking = true; } else { - sb.append(expr); + sb.append(HiveUtils.unparseIdentifier(colName, conf)); } } - } else { - for (int index = 0; index < privObject.getColumns().size(); index++) { - String expr = privObject.getColumns().get(index); - if (!firstOne) { - sb.append(", "); - } else { - firstOne = false; - } - sb.append(expr); + } + if (!doColumnMasking) { + sb = new StringBuilder(); + sb.append("(SELECT *"); + } + + if (!maskAndFilterInfo.isView) { + // put all virtual columns in RowResolver. + Iterator vcs = VirtualColumn.getRegistry(conf).iterator(); + while (vcs.hasNext()) { + VirtualColumn vc = vcs.next(); + sb.append(", " + vc.getName()); } } - sb.append(" FROM `" + privObject.getDbname() + "`.`" + privObject.getObjectName() + "`"); + + sb.append(" FROM "); + sb.append(HiveUtils.unparseIdentifier(privObject.getDbname(), conf)); + sb.append("."); + sb.append(HiveUtils.unparseIdentifier(privObject.getObjectName(), conf)); sb.append(" " + maskAndFilterInfo.additionalTabInfo); String filter = privObject.getRowFilterExpression(); if (filter != null) { sb.append(" WHERE " + filter); + doRowFiltering = true; + } + sb.append(")" + HiveUtils.unparseIdentifier(maskAndFilterInfo.alias, conf)); + + if (!doColumnMasking && !doRowFiltering) { + // nothing to do + return null; + } else { + LOG.debug("TableMask creates `" + sb.toString() + "`"); + return sb.toString(); } - sb.append(")" + maskAndFilterInfo.alias); - LOG.debug("TableMask creates `" + sb.toString() + "`"); - return sb.toString(); } - void addTableMasking(ASTNode node, String replacementText) throws SemanticException { + void addTranslation(ASTNode node, String replacementText) throws SemanticException { translator.addTranslation(node, replacementText); } - void applyTableMasking(TokenRewriteStream tokenRewriteStream) throws SemanticException { + void applyTranslations(TokenRewriteStream tokenRewriteStream) throws SemanticException { translator.applyTranslations(tokenRewriteStream); } diff --git a/ql/src/test/queries/clientpositive/masking_6.q b/ql/src/test/queries/clientpositive/masking_6.q new file mode 100644 index 0000000..e266741 --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_6.q @@ -0,0 +1,28 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, value, '12' from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; diff --git a/ql/src/test/queries/clientpositive/masking_7.q b/ql/src/test/queries/clientpositive/masking_7.q new file mode 100644 index 0000000..6d4784d --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_7.q @@ -0,0 +1,28 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; + +drop view masking_test; + +create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select * from masking_test; + +select * from masking_test; + +explain select * from masking_test where key > 0; + +select * from masking_test where key > 0; diff --git a/ql/src/test/queries/clientpositive/masking_8.q b/ql/src/test/queries/clientpositive/masking_8.q new file mode 100644 index 0000000..75d4447 --- /dev/null +++ b/ql/src/test/queries/clientpositive/masking_8.q @@ -0,0 +1,25 @@ +set hive.mapred.mode=nonstrict; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; + +drop table masking_test; + +create table masking_test as select cast(key as int) as key, value, '12' from src; + +explain select *, ROW__ID from masking_test; + +select *, ROW__ID from masking_test; + +explain select * from masking_test; + +select * from masking_test; + +drop table masking_test; + +create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src; + +explain select *, ROW__ID from masking_test; + +select *, ROW__ID from masking_test; + diff --git a/ql/src/test/queries/clientpositive/view_alias.q b/ql/src/test/queries/clientpositive/view_alias.q new file mode 100644 index 0000000..e85ba12 --- /dev/null +++ b/ql/src/test/queries/clientpositive/view_alias.q @@ -0,0 +1,34 @@ +drop view v; +create view v as select key, '12' from src; +desc formatted v; +select * from v order by `_c1` limit 5; + +drop view v; +create view v as select key as `_c1`, '12' from src; +desc formatted v; +select * from v order by `_c1` limit 5; + +drop view v; +create view v as select *, '12' from src; +desc formatted v; +select * from v order by `_c2` limit 5; + +drop view v; +create view v as select *, '12' as `_c121` from src; +desc formatted v; +select * from v order by `_c121` limit 5; + +drop view v; +create view v as select key, count(*) from src group by key; +desc formatted v; +select * from v order by `_c1` limit 5; + + +drop view v; +create table a (ca string, caa string); +create table b (cb string, cbb string); +insert into a select * from src limit 5; +insert into b select * from src limit 5; +create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb; +desc formatted v; +select * from v order by `_c3` limit 5; diff --git a/ql/src/test/results/clientpositive/masking_1.q.out b/ql/src/test/results/clientpositive/masking_1.q.out index 3b63550..69382b2 100644 --- a/ql/src/test/results/clientpositive/masking_1.q.out +++ b/ql/src/test/results/clientpositive/masking_1.q.out @@ -76,19 +76,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -123,19 +126,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -170,19 +176,26 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: reverse(value) (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + expressions: key (type: int), reverse(value) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -223,44 +236,47 @@ STAGE PLANS: expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Filter Operator + predicate: UDFToDouble(_col0) is not null (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -350,19 +366,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -397,19 +416,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -444,19 +466,25 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) > 0.0) and (UDFToDouble(key) < 10.0) and ((UDFToDouble(key) % 2.0) = 0.0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), upper(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 < 10) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 % 2) = 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_1_newdb.q.out b/ql/src/test/results/clientpositive/masking_1_newdb.q.out index 51c2619..b28c7f8 100644 --- a/ql/src/test/results/clientpositive/masking_1_newdb.q.out +++ b/ql/src/test/results/clientpositive/masking_1_newdb.q.out @@ -94,19 +94,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_2.q.out b/ql/src/test/results/clientpositive/masking_2.q.out index ff045a9..55ebaff 100644 --- a/ql/src/test/results/clientpositive/masking_2.q.out +++ b/ql/src/test/results/clientpositive/masking_2.q.out @@ -23,20 +23,24 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -75,20 +79,27 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -122,53 +133,63 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col1 is not null (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0) and reverse(value) is not null) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) + value expressions: value (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: - 0 _col0 (type: string) + 0 key (type: string) 1 _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + outputColumnNames: _col0, _col1, _col5, _col6 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col5 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int), _col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -191,35 +212,49 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + Filter Operator + predicate: UDFToDouble(_col0) is not null (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int), _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0) and reverse(value) is not null) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col1) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col1) (type: double) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(_col1) is not null (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col1) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col1) (type: double) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) Reduce Operator Tree: Join Operator condition map: @@ -229,13 +264,16 @@ STAGE PLANS: 1 UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col2 > 0) (type: boolean) + Statistics: Num rows: 30 Data size: 319 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 319 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -258,47 +296,58 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Union Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10) and (UDFToInteger(key) > 0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(key) (type: int), reverse(value) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - Union - Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Union Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 110 Data size: 1167 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: string) diff --git a/ql/src/test/results/clientpositive/masking_3.q.out b/ql/src/test/results/clientpositive/masking_3.q.out index 1925dce..044648e 100644 --- a/ql/src/test/results/clientpositive/masking_3.q.out +++ b/ql/src/test/results/clientpositive/masking_3.q.out @@ -28,52 +28,54 @@ STAGE PLANS: Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -607,54 +609,59 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1185,54 +1192,59 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: double), _col2 (type: double) - sort order: ++ - Map-reduce partition columns: _col1 (type: double), _col2 (type: double) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int) + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col1 (type: double), _col2 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1763,54 +1775,63 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: double), _col2 (type: double) - sort order: ++ - Map-reduce partition columns: _col1 (type: double), _col2 (type: double) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col1 (type: double), _col2 (type: double) - 1 _col0 (type: double), _col1 (type: double) - outputColumnNames: _col0 + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -2342,53 +2363,58 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(key) is not null and key is not null) (type: boolean) + predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Filter Operator + predicate: (1 = 1) (type: boolean) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: UDFToDouble(_col0) is not null (type: boolean) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -2404,34 +2430,34 @@ STAGE PLANS: alias: srcpart Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) + predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(key) (type: double) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + value expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 UDFToDouble(_col0) (type: double) - 1 UDFToDouble(_col0) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + 1 UDFToDouble(key) (type: double) + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -6587,54 +6613,59 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7165,54 +7196,59 @@ STAGE PLANS: alias: masking_test_subq Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key > 0) and UDFToDouble(key) is not null) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + predicate: UDFToDouble(key) is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 183 Data size: 1944 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -7743,19 +7779,25 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((UDFToDouble(key) > 0.0) and (UDFToDouble(key) < 10.0) and ((UDFToDouble(key) % 2.0) = 0.0)) (type: boolean) - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + predicate: (key > 0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), upper(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 < 10) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 % 2) = 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_4.q.out b/ql/src/test/results/clientpositive/masking_4.q.out index 7e923e8..b8c0a92 100644 --- a/ql/src/test/results/clientpositive/masking_4.q.out +++ b/ql/src/test/results/clientpositive/masking_4.q.out @@ -47,16 +47,19 @@ STAGE PLANS: predicate: (key = '5') (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '5' (type: string) + expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: (_col0 = '5') (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -88,19 +91,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key = 5)) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 5 (type: int), reverse(value) (type: string) + expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = 5) (type: boolean) Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -132,19 +138,22 @@ STAGE PLANS: alias: masking_test Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key % 2) = 0) and (key < 10) and (key = 5)) (type: boolean) - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 5 (type: int), reverse(value) (type: string) + expressions: key (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 = 5) (type: boolean) Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -178,52 +187,54 @@ STAGE PLANS: Filter Operator predicate: UDFToDouble(key) is not null (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: double), _col3 (type: double) - sort order: ++ - Map-reduce partition columns: _col2 (type: double), _col3 (type: double) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: key (type: int), value (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: UDFToDouble(key) is not null (type: boolean) + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + expressions: key (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: _col0 (type: double), _col1 (type: double) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - sort order: ++ - Map-reduce partition columns: _col0 (type: double), _col1 (type: double) + Filter Operator + predicate: (UDFToDouble(_col0) is not null and UDFToDouble(_col1) is not null) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + sort order: ++ + Map-reduce partition columns: UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Left Semi Join 0 to 1 keys: - 0 _col2 (type: double), _col3 (type: double) - 1 _col0 (type: double), _col1 (type: double) + 0 UDFToDouble(key) (type: double), UDFToDouble(key) (type: double) + 1 UDFToDouble(_col0) (type: double), UDFToDouble(_col1) (type: double) outputColumnNames: _col0, _col1 Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Filter Operator + predicate: (1 = 1) (type: boolean) Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_6.q.out b/ql/src/test/results/clientpositive/masking_6.q.out new file mode 100644 index 0000000..feed765 --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_6.q.out @@ -0,0 +1,256 @@ +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view masking_test as select cast(key as int) as key, value, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, value, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0_lav 12 +4 4_lav 12 +8 8_lav 12 +0 0_lav 12 +0 0_lav 12 +2 2_lav 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 4_lav 12 +8 8_lav 12 +2 2_lav 12 +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, '12', +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 12 12 12 12 12 12 12 12 12 12 12 +4 12 12 12 12 12 12 12 12 12 12 12 +8 12 12 12 12 12 12 12 12 12 12 12 +0 12 12 12 12 12 12 12 12 12 12 12 +0 12 12 12 12 12 12 12 12 12 12 12 +2 12 12 12 12 12 12 12 12 12 12 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 12 12 12 12 12 12 12 12 12 12 12 +8 12 12 12 12 12 12 12 12 12 12 12 +2 12 12 12 12 12 12 12 12 12 12 12 diff --git a/ql/src/test/results/clientpositive/masking_7.q.out b/ql/src/test/results/clientpositive/masking_7.q.out new file mode 100644 index 0000000..735f17b --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_7.q.out @@ -0,0 +1,256 @@ +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, value, '12', ROW__ID from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string), '12' (type: string), _col3 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0_lav 12 NULL +4 4_lav 12 NULL +8 8_lav 12 NULL +0 0_lav 12 NULL +0 0_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string), _col3 (type: struct) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), '12' (type: string), _col3 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 4_lav 12 NULL +8 8_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: drop view masking_test +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop view masking_test +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create view masking_test as select cast(key as int) as key, '12', ROW__ID, +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), ROW__ID (type: struct), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +4 12 NULL 12 12 12 12 12 12 12 12 12 12 +8 12 NULL 12 12 12 12 12 12 12 12 12 12 +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +0 12 NULL 12 12 12 12 12 12 12 12 12 12 +2 12 NULL 12 12 12 12 12 12 12 12 12 12 +PREHOOK: query: explain select * from masking_test where key > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test where key > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(key) (type: int), '12' (type: string), ROW__ID (type: struct), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string), '12' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (_col0 > 0) (type: boolean) + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test where key > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test where key > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +POSTHOOK: Input: default@src +#### A masked pattern was here #### +4 12 NULL 12 12 12 12 12 12 12 12 12 12 +8 12 NULL 12 12 12 12 12 12 12 12 12 12 +2 12 NULL 12 12 12 12 12 12 12 12 12 12 diff --git a/ql/src/test/results/clientpositive/masking_8.q.out b/ql/src/test/results/clientpositive/masking_8.q.out new file mode 100644 index 0000000..f813a3e --- /dev/null +++ b/ql/src/test/results/clientpositive/masking_8.q.out @@ -0,0 +1,200 @@ +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table masking_test as select cast(key as int) as key, value, '12' from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test as select cast(key as int) as key, value, '12' from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +POSTHOOK: Lineage: masking_test.c2 SIMPLE [] +POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select *, ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), reverse(value) (type: string), c2 (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select *, ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +0 0_lav 12 NULL +4 4_lav 12 NULL +8 8_lav 12 NULL +0 0_lav 12 NULL +0 0_lav 12 NULL +2 2_lav 12 NULL +PREHOOK: query: explain select * from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), reverse(value) (type: string), c2 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select * from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +0 0_lav 12 +4 4_lav 12 +8 8_lav 12 +0 0_lav 12 +0 0_lav 12 +2 2_lav 12 +PREHOOK: query: drop table masking_test +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test +POSTHOOK: query: drop table masking_test +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test +PREHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@masking_test +POSTHOOK: query: create table masking_test as select cast(key as int) as key, '12' +'12', '12', '12', '12', '12', '12', '12', '12', '12', '12' + from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@masking_test +POSTHOOK: Lineage: masking_test.c1 SIMPLE [] +POSTHOOK: Lineage: masking_test.c10 SIMPLE [] +POSTHOOK: Lineage: masking_test.c2 SIMPLE [] +POSTHOOK: Lineage: masking_test.c3 SIMPLE [] +POSTHOOK: Lineage: masking_test.c4 SIMPLE [] +POSTHOOK: Lineage: masking_test.c5 SIMPLE [] +POSTHOOK: Lineage: masking_test.c6 SIMPLE [] +POSTHOOK: Lineage: masking_test.c7 SIMPLE [] +POSTHOOK: Lineage: masking_test.c8 SIMPLE [] +POSTHOOK: Lineage: masking_test.c9 SIMPLE [] +POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain select *, ROW__ID from masking_test +PREHOOK: type: QUERY +POSTHOOK: query: explain select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: masking_test + Statistics: Num rows: 500 Data size: 17406 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), c1 (type: string), c2 (type: string), c3 (type: string), c4 (type: string), c5 (type: string), c6 (type: string), c7 (type: string), c8 (type: string), c9 (type: string), c10 (type: string), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 2889 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select *, ROW__ID from masking_test +PREHOOK: type: QUERY +PREHOOK: Input: default@masking_test +#### A masked pattern was here #### +POSTHOOK: query: select *, ROW__ID from masking_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@masking_test +#### A masked pattern was here #### +0 1212 12 12 12 12 12 12 12 12 12 NULL +4 1212 12 12 12 12 12 12 12 12 12 NULL +8 1212 12 12 12 12 12 12 12 12 12 NULL +0 1212 12 12 12 12 12 12 12 12 12 NULL +0 1212 12 12 12 12 12 12 12 12 12 NULL +2 1212 12 12 12 12 12 12 12 12 12 NULL diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out index 6717527..1dc2461 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out @@ -248,15 +248,19 @@ STAGE PLANS: keys: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out index 6aaab20..83cb6ca 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_3.q.out @@ -2403,15 +2403,19 @@ STAGE PLANS: keys: 0 UDFToDouble(_col0) (type: double) 1 UDFToDouble(key) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/view_alias.q.out b/ql/src/test/results/clientpositive/view_alias.q.out new file mode 100644 index 0000000..78ff5e2 --- /dev/null +++ b/ql/src/test/results/clientpositive/view_alias.q.out @@ -0,0 +1,427 @@ +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select key, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key, '12' from src +View Expanded Text: select `src`.`key`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 12 +27 12 +311 12 +97 12 +238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select key as `_c1`, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key as `_c1`, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key as _c1, '12' from src +View Expanded Text: select `src`.`key` as `_c1`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 12 +27 12 +311 12 +97 12 +238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select *, '12' from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select *, '12' from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string +_c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select *, '12' from src +View Expanded Text: select `src`.`key`, `src`.`value`, '12' from `default`.`src` +PREHOOK: query: select * from v order by `_c2` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c2` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 val_165 12 +27 val_27 12 +311 val_311 12 +97 val_97 12 +238 val_238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select *, '12' as `_c121` from src +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select *, '12' as `_c121` from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +value string +_c121 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select *, '12' as _c121 from src +View Expanded Text: select `src`.`key`, `src`.`value`, '12' as `_c121` from `default`.`src` +PREHOOK: query: select * from v order by `_c121` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c121` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +165 val_165 12 +27 val_27 12 +311 val_311 12 +97 val_97 12 +238 val_238 12 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select key, count(*) from src group by key +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select key, count(*) from src group by key +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +key string +_c1 bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key, count(*) from src group by key +View Expanded Text: select `src`.`key`, count(*) from `default`.`src` group by `src`.`key` +PREHOOK: query: select * from v order by `_c1` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c1` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +11 1 +105 1 +114 1 +96 1 +10 1 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create table a (ca string, caa string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a (ca string, caa string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b (cb string, cbb string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b (cb string, cbb string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: insert into a select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +POSTHOOK: query: insert into a select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Lineage: a.ca SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.caa SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into b select * from src limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@b +POSTHOOK: query: insert into b select * from src limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@b +POSTHOOK: Lineage: b.cb SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.cbb SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@a +PREHOOK: Input: default@b +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@a +POSTHOOK: Input: default@b +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +_c0 string +ca string +caa string +_c3 int +cb string +cbb string +_c6 int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select '010', a.*, 121, b.*, 234 from a join b on a.ca = b.cb +View Expanded Text: select '010', `a`.`ca`, `a`.`caa`, 121, `b`.`cb`, `b`.`cbb`, 234 from `default`.`a` join `default`.`b` on `a`.`ca` = `b`.`cb` +PREHOOK: query: select * from v order by `_c3` limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@a +PREHOOK: Input: default@b +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v order by `_c3` limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@a +POSTHOOK: Input: default@b +POSTHOOK: Input: default@v +#### A masked pattern was here #### +010 86 val_86 121 86 val_86 234 +010 311 val_311 121 311 val_311 234 +010 27 val_27 121 27 val_27 234 +010 238 val_238 121 238 val_238 234 +010 165 val_165 121 165 val_165 234