diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/NamedJoinInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/NamedJoinInfo.java new file mode 100644 index 0000000..338fac3 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/NamedJoinInfo.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.util.List; + +import org.apache.hadoop.hive.ql.parse.JoinType; + +public class NamedJoinInfo { + private List aliases; + private List namedColumns; + private JoinType hiveJoinType; + + public NamedJoinInfo(List aliases, List namedColumns, JoinType hiveJoinType) { + super(); + this.aliases = aliases; + this.namedColumns = namedColumns; + this.hiveJoinType = hiveJoinType; + } + + public List getAliases() { + return aliases; + } + + public void setAliases(List aliases) { + this.aliases = aliases; + } + + public List getNamedColumns() { + return namedColumns; + } + + public void setNamedColumns(List namedColumns) { + this.namedColumns = namedColumns; + } + + public JoinType getHiveJoinType() { + return hiveJoinType; + } + + public void setHiveJoinType(JoinType hiveJoinType) { + this.hiveJoinType = hiveJoinType; + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index bf0a11b..c5a9af4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -127,6 +127,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.NamedJoinInfo; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.RowSchema; @@ -205,6 +206,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewFilterScanRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinCondTypeCheckProcFactory; @@ -1907,7 +1909,7 @@ private RelNode genSetOpLogicalPlan(Opcode opcode, String alias, String leftalia return setOpRel; } - private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJoinType, + private RelNode genJoinRelNode(RelNode leftRel, String leftTableAlias, RelNode rightRel, String rightTableAlias, JoinType hiveJoinType, ASTNode joinCond) throws SemanticException { RowResolver leftRR = this.relToHiveRR.get(leftRel); @@ -1915,10 +1917,39 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ // 1. Construct ExpressionNodeDesc representing Join Condition RexNode calciteJoinCond = null; + List namedColumns = null; if (joinCond != null) { JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); RowResolver input = RowResolver.getCombinedRR(leftRR, rightRR); - if (unparseTranslator != null && unparseTranslator.isEnabled()) { + // named columns join + // TODO: we can also do the same for semi join but it seems that other + // DBMS does not support it yet. + if (joinCond.getType() == HiveParser.TOK_TABCOLNAME + && !hiveJoinType.equals(JoinType.LEFTSEMI)) { + namedColumns = new ArrayList<>(); + ASTNode and = (ASTNode) ParseDriver.adaptor.create(HiveParser.KW_AND, "and"); + ASTNode equal = null; + int count = 0; + for (Node child : joinCond.getChildren()) { + String columnName = ((ASTNode) child).getText(); + // dealing with views + if (unparseTranslator != null && unparseTranslator.isEnabled()) { + unparseTranslator.addIdentifierTranslation((ASTNode) child); + } + namedColumns.add(columnName); + ASTNode left = ASTBuilder.qualifiedName(leftTableAlias, columnName); + ASTNode right = ASTBuilder.qualifiedName(rightTableAlias, columnName); + equal = (ASTNode) ParseDriver.adaptor.create(HiveParser.EQUAL, "="); + ParseDriver.adaptor.addChild(equal, left); + ParseDriver.adaptor.addChild(equal, right); + ParseDriver.adaptor.addChild(and, equal); + count++; + } + if (count == 0) { + throw new SemanticException("syntax error at or near named columns join"); + } + joinCond = count > 1 ? and : equal; + } else if (unparseTranslator != null && unparseTranslator.isEnabled()) { genAllExprNodeDesc(joinCond, input, jCtx); } Map exprNodes = JoinCondTypeCheckProcFactory.genExprNode(joinCond, @@ -2032,12 +2063,17 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ } else { topRel = HiveJoin.getJoin(cluster, leftRel, rightRel, calciteJoinCond, calciteJoinType); topRR = RowResolver.getCombinedRR(leftRR, rightRR); + if (namedColumns != null) { + List tableAliases = new ArrayList<>(); + tableAliases.add(leftTableAlias); + tableAliases.add(rightTableAlias); + topRR.setNamedJoinInfo(new NamedJoinInfo(tableAliases, namedColumns, hiveJoinType)); + } } // 4. Add new rel & its RR to the maps relToHiveColNameCalcitePosMap.put(topRel, this.buildHiveToCalciteColumnMap(topRR, topRel)); relToHiveRR.put(topRel, topRR); - return topRel; } @@ -2086,12 +2122,13 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // 2. Get Left Table Alias ASTNode left = (ASTNode) joinParseTree.getChild(0); + String leftTableAlias = null; if ((left.getToken().getType() == HiveParser.TOK_TABREF) || (left.getToken().getType() == HiveParser.TOK_SUBQUERY) || (left.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { String tableName = SemanticAnalyzer.getUnescapedUnqualifiedTableName( (ASTNode) left.getChild(0)).toLowerCase(); - String leftTableAlias = left.getChildCount() == 1 ? tableName : SemanticAnalyzer + leftTableAlias = left.getChildCount() == 1 ? tableName : SemanticAnalyzer .unescapeIdentifier(left.getChild(left.getChildCount() - 1).getText().toLowerCase()); // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) @@ -2107,12 +2144,13 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a // 3. Get Right Table Alias ASTNode right = (ASTNode) joinParseTree.getChild(1); + String rightTableAlias = null; if ((right.getToken().getType() == HiveParser.TOK_TABREF) || (right.getToken().getType() == HiveParser.TOK_SUBQUERY) || (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) { String tableName = SemanticAnalyzer.getUnescapedUnqualifiedTableName( (ASTNode) right.getChild(0)).toLowerCase(); - String rightTableAlias = right.getChildCount() == 1 ? tableName : SemanticAnalyzer + rightTableAlias = right.getChildCount() == 1 ? tableName : SemanticAnalyzer .unescapeIdentifier(right.getChild(right.getChildCount() - 1).getText().toLowerCase()); // ptf node form is: ^(TOK_PTBLFUNCTION $name $alias? // partitionTableFunctionSource partitioningSpec? expression*) @@ -2128,7 +2166,7 @@ private RelNode genJoinLogicalPlan(ASTNode joinParseTree, Map a ASTNode joinCond = (ASTNode) joinParseTree.getChild(2); // 5. Create Join rel - return genJoinRelNode(leftRel, rightRel, hiveJoinType, joinCond); + return genJoinRelNode(leftRel, leftTableAlias, rightRel, rightTableAlias, hiveJoinType, joinCond); } private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index c8c71b0..e8ea443 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -110,9 +110,15 @@ atomjoinSource joinSource : - atomjoinSource (joinToken^ joinSourcePart (KW_ON! expression {$joinToken.start.getType() != COMMA}?)?)* + atomjoinSource (joinToken^ joinSourcePart (joinCondition)?)* ; +joinCondition + : + KW_ON! expression + | + KW_USING! columnParenthesesList + ; joinSourcePart @init { gParent.pushMsg("joinSourcePart", state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 37817ce..d98a663 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -2145,7 +2145,7 @@ columnNameOrderList columnParenthesesList @init { pushMsg("column parentheses list", state); } @after { popMsg(state); } - : LPAREN columnNameList RPAREN + : LPAREN! columnNameList RPAREN! ; enableSpecification diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index e14f1cf..eb333c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.NamedJoinInfo; import org.apache.hadoop.hive.ql.exec.RowSchema; /** @@ -56,6 +57,8 @@ private static final Logger LOG = LoggerFactory.getLogger(RowResolver.class.getName()); + private NamedJoinInfo namedJoinInfo; + public RowResolver() { rowSchema = new RowSchema(); rslvMap = new LinkedHashMap>(); @@ -469,4 +472,12 @@ public RowResolver duplicate() { private HashMap getInvRslvMap() { return invRslvMap; // If making this public, note that its ordering is undefined. } + + public NamedJoinInfo getNamedJoinInfo() { + return namedJoinInfo; + } + + public void setNamedJoinInfo(NamedJoinInfo namedJoinInfo) { + this.namedJoinInfo = namedJoinInfo; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0732207..769d3d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3427,6 +3427,67 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, Map inputColsProcessed = new HashMap(); // For expr "*", aliases should be iterated in the order they are specified // in the query. + + if (colSrcRR.getNamedJoinInfo() != null) { + HashMap leftMap = colSrcRR.getFieldMap(colSrcRR.getNamedJoinInfo().getAliases().get(0)); + HashMap rightMap = colSrcRR.getFieldMap(colSrcRR.getNamedJoinInfo().getAliases().get(1)); + HashMap chosenMap = null; + if (colSrcRR.getNamedJoinInfo().getHiveJoinType() != JoinType.RIGHTOUTER) { + chosenMap = leftMap; + } else { + chosenMap = rightMap; + } + // first get the columns in named columns + for (String columnName : colSrcRR.getNamedJoinInfo().getNamedColumns()) { + for (Map.Entry entry : chosenMap.entrySet()) { + ColumnInfo colInfo = entry.getValue(); + if (!columnName.equals(colInfo.getAlias())) { + continue; + } + String name = colInfo.getInternalName(); + String[] tmp = colSrcRR.reverseLookup(name); + + // Skip the colinfos which are not for this particular alias + if (tabAlias != null && !tmp[0].equalsIgnoreCase(tabAlias)) { + continue; + } + + if (colInfo.getIsVirtualCol() && colInfo.isHiddenVirtualCol()) { + continue; + } + ColumnInfo oColInfo = inputColsProcessed.get(colInfo); + if (oColInfo == null) { + ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), name, + colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isSkewedCol()); + col_list.add(expr); + oColInfo = new ColumnInfo(getColumnInternalName(pos), colInfo.getType(), + colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); + inputColsProcessed.put(colInfo, oColInfo); + } + if (ensureUniqueCols) { + if (!output.putWithCheck(tmp[0], tmp[1], null, oColInfo)) { + throw new CalciteSemanticException("Cannot add column to RR: " + tmp[0] + "." + + tmp[1] + " => " + oColInfo + " due to duplication, see previous warnings", + UnsupportedFeature.Duplicates_in_RR); + } + } else { + output.put(tmp[0], tmp[1], oColInfo); + } + pos = Integer.valueOf(pos.intValue() + 1); + matched++; + + if (unparseTranslator.isEnabled() || tableMask.isEnabled()) { + if (replacementText.length() > 0) { + replacementText.append(", "); + } + replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf)); + replacementText.append("."); + replacementText.append(HiveUtils.unparseIdentifier(tmp[1], conf)); + } + } + } + } + // then get the columns not in named columns for (String alias : aliases) { HashMap fMap = colSrcRR.getFieldMap(alias); if (fMap == null) { @@ -3436,12 +3497,18 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, // from the input schema for (Map.Entry entry : fMap.entrySet()) { ColumnInfo colInfo = entry.getValue(); + if (colSrcRR.getNamedJoinInfo() != null && colSrcRR.getNamedJoinInfo().getNamedColumns().contains(colInfo.getAlias())) { + continue; + } if (excludeCols != null && excludeCols.contains(colInfo)) { continue; // This was added during plan generation. } - // First, look up the column from the source against which * is to be resolved. - // We'd later translated this into the column from proper input, if it's valid. - // TODO: excludeCols may be possible to remove using the same technique. + // First, look up the column from the source against which * is to be + // resolved. + // We'd later translated this into the column from proper input, if + // it's valid. + // TODO: excludeCols may be possible to remove using the same + // technique. String name = colInfo.getInternalName(); String[] tmp = colSrcRR.reverseLookup(name); @@ -3459,13 +3526,14 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, continue; } - // If input (GBY) is different than the source of columns, find the same column in input. + // If input (GBY) is different than the source of columns, find the + // same column in input. // TODO: This is fraught with peril. if (input != colSrcRR) { colInfo = input.get(tabAlias, tmp[1]); if (colInfo == null) { - LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1] - + ", derived from [" + colSrcRR + "], in [" + input + "]"); + LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1] + ", derived from [" + + colSrcRR + "], in [" + input + "]"); throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY, tmp[1]); } String oldCol = null; @@ -3482,13 +3550,11 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, ColumnInfo oColInfo = inputColsProcessed.get(colInfo); if (oColInfo == null) { - ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), - name, colInfo.getTabAlias(), colInfo.getIsVirtualCol(), - colInfo.isSkewedCol()); + ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(), name, + colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isSkewedCol()); col_list.add(expr); - oColInfo = new ColumnInfo(getColumnInternalName(pos), - colInfo.getType(), colInfo.getTabAlias(), - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); + oColInfo = new ColumnInfo(getColumnInternalName(pos), colInfo.getType(), + colInfo.getTabAlias(), colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); inputColsProcessed.put(colInfo, oColInfo); } if (ensureUniqueCols) { @@ -3513,6 +3579,7 @@ Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel, } } } + if (matched == 0) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(sel)); } diff --git a/ql/src/test/queries/clientpositive/named_column_join.q b/ql/src/test/queries/clientpositive/named_column_join.q new file mode 100644 index 0000000..4f117d8 --- /dev/null +++ b/ql/src/test/queries/clientpositive/named_column_join.q @@ -0,0 +1,45 @@ +set hive.mapred.mode=nonstrict; +-- SORT_QUERY_RESULTS + +create table t (a int, b int); + +insert into t values (1,2),(2,1),(3,4),(4,3),(3,3),(null,null),(null,1),(2,null); + +explain select * from t t1 join t t2 using (a); + +select * from t t1 join t t2 using (a); + +select * from t t1 join t t2 using (a,b); + +select t1.a,t2.b,t1.b,t2.a from t t1 join t t2 using (a); + +select * from t t1 left outer join t t2 using (a,b); + +select t1.a,t1.b from t t1 right outer join t t2 on (t1.a=t2.a and t1.b=t2.b); + +select * from t t1 right outer join t t2 using (a,b); + +select * from t t1 inner join t t2 using (a,b); + +select * from t t1 left outer join t t2 using (b); + +select * from t t1 right outer join t t2 using (b); + +select * from t t1 inner join t t2 using (b); + +drop view v; + +create view v as select * from t t1 join t t2 using (a,b); + +desc formatted v; + +select * from v; + +drop view v; + +create view v as select * from t t1 right outer join t t2 using (b,a); + +desc formatted v; + +select * from v; + diff --git a/ql/src/test/results/clientpositive/named_column_join.q.out b/ql/src/test/results/clientpositive/named_column_join.q.out new file mode 100644 index 0000000..568df63 --- /dev/null +++ b/ql/src/test/results/clientpositive/named_column_join.q.out @@ -0,0 +1,388 @@ +PREHOOK: query: create table t (a int, b int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (a int, b int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values (1,2),(2,1),(3,4),(4,3),(3,3),(null,null),(null,1),(2,null) +PREHOOK: type: QUERY +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values (1,2),(2,1),(3,4),(4,3),(3,3),(null,null),(null,1),(2,null) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: t.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: explain select * from t t1 join t t2 using (a) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from t t1 join t t2 using (a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + TableScan + alias: t2 + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col3 + Statistics: Num rows: 8 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 8 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 30 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from t t1 join t t2 using (a) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 join t t2 using (a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 2 +2 1 1 +2 1 NULL +2 NULL 1 +2 NULL NULL +3 3 3 +3 3 4 +3 4 3 +3 4 4 +4 3 3 +PREHOOK: query: select * from t t1 join t t2 using (a,b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 join t t2 using (a,b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 +2 1 +3 3 +3 4 +4 3 +PREHOOK: query: select t1.a,t2.b,t1.b,t2.a from t t1 join t t2 using (a) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select t1.a,t2.b,t1.b,t2.a from t t1 join t t2 using (a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 2 1 +2 1 1 2 +2 1 NULL 2 +2 NULL 1 2 +2 NULL NULL 2 +3 3 3 3 +3 3 4 3 +3 4 3 3 +3 4 4 3 +4 3 3 4 +PREHOOK: query: select * from t t1 left outer join t t2 using (a,b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 left outer join t t2 using (a,b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 +2 1 +2 NULL +3 3 +3 4 +4 3 +NULL 1 +NULL NULL +PREHOOK: query: select t1.a,t1.b from t t1 right outer join t t2 on (t1.a=t2.a and t1.b=t2.b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select t1.a,t1.b from t t1 right outer join t t2 on (t1.a=t2.a and t1.b=t2.b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 +2 1 +3 3 +3 4 +4 3 +NULL NULL +NULL NULL +NULL NULL +PREHOOK: query: select * from t t1 right outer join t t2 using (a,b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 right outer join t t2 using (a,b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 +2 1 +2 NULL +3 3 +3 4 +4 3 +NULL 1 +NULL NULL +PREHOOK: query: select * from t t1 inner join t t2 using (a,b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 inner join t t2 using (a,b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 +2 1 +3 3 +3 4 +4 3 +PREHOOK: query: select * from t t1 left outer join t t2 using (b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 left outer join t t2 using (b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 2 +1 2 NULL +1 NULL 2 +1 NULL NULL +2 1 1 +3 3 3 +3 3 4 +3 4 3 +3 4 4 +4 3 3 +NULL 2 NULL +NULL NULL NULL +PREHOOK: query: select * from t t1 right outer join t t2 using (b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 right outer join t t2 using (b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 2 +1 2 NULL +1 NULL 2 +1 NULL NULL +2 1 1 +3 3 3 +3 3 4 +3 4 3 +3 4 4 +4 3 3 +NULL NULL 2 +NULL NULL NULL +PREHOOK: query: select * from t t1 inner join t t2 using (b) +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t t1 inner join t t2 using (b) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 2 2 +1 2 NULL +1 NULL 2 +1 NULL NULL +2 1 1 +3 3 3 +3 3 4 +3 4 3 +3 4 4 +4 3 3 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select * from t t1 join t t2 using (a,b) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select * from t t1 join t t2 using (a,b) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.a SIMPLE [(t)t1.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: v.b SIMPLE [(t)t1.FieldSchema(name:b, type:int, comment:null), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +a int +b int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from t t1 join t t2 using (a,b) +View Expanded Text: select `t1`.`a`, `t1`.`b` from `default`.`t` `t1` join `default`.`t` `t2` using (`a`,`b`) +View Rewrite Enabled: No +PREHOOK: query: select * from v +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Input: default@v +#### A masked pattern was here #### +1 2 +2 1 +3 3 +3 4 +4 3 +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as select * from t t1 right outer join t t2 using (b,a) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as select * from t t1 right outer join t t2 using (b,a) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.a SIMPLE [(t)t2.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: v.b SIMPLE [(t)t2.FieldSchema(name:b, type:int, comment:null), ] +PREHOOK: query: desc formatted v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: desc formatted v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +# col_name data_type comment + +b int +a int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from t t1 right outer join t t2 using (b,a) +View Expanded Text: select `t2`.`b`, `t2`.`a` from `default`.`t` `t1` right outer join `default`.`t` `t2` using (`b`,`a`) +View Rewrite Enabled: No +PREHOOK: query: select * from v +PREHOOK: type: QUERY +PREHOOK: Input: default@t +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +POSTHOOK: Input: default@v +#### A masked pattern was here #### +1 2 +1 NULL +2 1 +3 3 +3 4 +4 3 +NULL 2 +NULL NULL