diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index bf0a11b..63ffa84 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2825,7 +2825,7 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException && selExprList.getChildCount() == 1 && selExprList.getChild(0).getChildCount() == 1) { ASTNode node = (ASTNode) selExprList.getChild(0).getChild(0); if (node.getToken().getType() == HiveParser.TOK_ALLCOLREF) { - srcRel = genSelectLogicalPlan(qb, srcRel, srcRel); + srcRel = genSelectLogicalPlan(qb, srcRel, srcRel, false).getKey(); RowResolver rr = this.relToHiveRR.get(srcRel); qbp.setSelExprForClause(detsClauseName, SemanticAnalyzer.genSelectDIAST(rr)); } @@ -2991,9 +2991,13 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException * top constraining Select * @throws SemanticException */ - private Pair genOBLogicalPlan(QB qb, RelNode srcRel, boolean outermostOB) - throws SemanticException { + private Pair genOBLogicalPlan(QB qb, Pair selPair, + boolean outermostOB) throws SemanticException { + // selPair.getKey() is the operator right before OB + // selPair.getValue() is the original RR that should be OB's RR + RelNode srcRel = selPair.getKey(); RelNode sortRel = null; + RelNode returnRel = null; RelNode originalOBChild = null; QBParseInfo qbp = getQBParseInfo(qb); @@ -3001,13 +3005,16 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException ASTNode obAST = qbp.getOrderByForClause(dest); if (obAST != null) { - // 1. OB Expr sanity test - // in strict mode, in the presence of order by, limit must be specified - Integer limit = qb.getParseInfo().getDestLimit(dest); - if (limit == null) { - String error = StrictChecks.checkNoLimit(conf); - if (error != null) { - throw new SemanticException(SemanticAnalyzer.generateErrorMessage(obAST, error)); + if (selPair.getValue() == null) { + // 1. OB Expr sanity test + // in strict mode, in the presence of order by, limit must be + // specified + Integer limit = qb.getParseInfo().getDestLimit(dest); + if (limit == null) { + String error = StrictChecks.checkNoLimit(conf); + if (error != null) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(obAST, error)); + } } } @@ -3064,8 +3071,8 @@ private RelNode genGBLogicalPlan(QB qb, RelNode srcRel) throws SemanticException } else if (nullObASTExpr.getType() == HiveParser.TOK_NULLS_LAST) { nullOrder = RelFieldCollation.NullDirection.LAST; } else { - throw new SemanticException( - "Unexpected null ordering option: " + nullObASTExpr.getType()); + throw new SemanticException("Unexpected null ordering option: " + + nullObASTExpr.getType()); } // 2.5 Add to field collations @@ -3137,9 +3144,25 @@ public RexNode apply(RelDataTypeField input) { outputRR, sortRel); relToHiveRR.put(sortRel, outputRR); relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap); - } - return (new Pair(sortRel, originalOBChild)); + if (selPair.getValue() != null) { + List originalInputRefs = Lists.transform(srcRel.getRowType().getFieldList(), + new Function() { + @Override + public RexNode apply(RelDataTypeField input) { + return new RexInputRef(input.getIndex(), input.getType()); + } + }); + List selectedRefs = Lists.newArrayList(); + for (int index = 0; index < selPair.getValue().getColumnInfos().size(); index++) { + selectedRefs.add(originalInputRefs.get(index)); + } + returnRel = genSelectRelNode(selectedRefs, selPair.getValue(), sortRel); + } else { + returnRel = sortRel; + } + } + return (new Pair(returnRel, originalOBChild)); } private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticException { @@ -3452,7 +3475,7 @@ private RelNode genSelectRelNode(List calciteColLst, RowResolver out_rw * * @throws SemanticException */ - private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) + private Pair genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel, boolean considerOB) throws SemanticException { // 0. Generate a Select Node for Windowing // Exclude the newly-generated select columns from */etc. resolution. @@ -3698,15 +3721,62 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) // 8. Build Calcite Rel RelNode outputRel = null; if (genericUDTF != null) { - // The basic idea for CBO support of UDTF is to treat UDTF as a special project. - // In AST return path, as we just need to generate a SEL_EXPR, we just need to remember the expressions and the alias. - // In OP return path, we need to generate a SEL and then a UDTF following old semantic analyzer. - outputRel = genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias, udtfColAliases, qb, calciteColLst, out_rwsch, srcRel); - } - else{ - outputRel = genSelectRelNode(calciteColLst, out_rwsch, srcRel); + // The basic idea for CBO support of UDTF is to treat UDTF as a special + // project. + // In AST return path, as we just need to generate a SEL_EXPR, we just + // need to remember the expressions and the alias. + // In OP return path, we need to generate a SEL and then a UDTF + // following old semantic analyzer. + outputRel = genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias, udtfColAliases, qb, + calciteColLst, out_rwsch, srcRel); + } else { + String dest = qbp.getClauseNames().iterator().next(); + ASTNode obAST = qbp.getOrderByForClause(dest); + + RowResolver originalRR = null; + // We only support unselected column following by order by. + // TODO: support unselected columns in genericUDTF and windowing functions. + if (obAST != null && !(selForWindow != null && selExprList.getToken().getType() == HiveParser.TOK_SELECTDI)) { + // 1. OB Expr sanity test + // in strict mode, in the presence of order by, limit must be + // specified + Integer limit = qb.getParseInfo().getDestLimit(dest); + if (limit == null) { + String error = StrictChecks.checkNoLimit(conf); + if (error != null) { + throw new SemanticException(SemanticAnalyzer.generateErrorMessage(obAST, error)); + } + } + List originalInputRefs = Lists.transform(srcRel.getRowType().getFieldList(), + new Function() { + @Override + public RexNode apply(RelDataTypeField input) { + return new RexInputRef(input.getIndex(), input.getType()); + } + }); + originalRR = out_rwsch.duplicate(); + for (int i = 0; i < inputRR.getColumnInfos().size(); i++) { + ColumnInfo colInfo = new ColumnInfo(inputRR.getColumnInfos().get(i)); + String internalName = SemanticAnalyzer.getColumnInternalName(out_rwsch.getColumnInfos() + .size() + i); + colInfo.setInternalName(internalName); + if (!out_rwsch.putWithCheck(colInfo.getTabAlias(), colInfo.getAlias(), internalName, + colInfo)) { + LOG.info("There is confict when we add column to RR for order by: " + + colInfo.getTabAlias() + "." + colInfo.getAlias() + " => " + colInfo + + " due to duplication, see previous warnings"); + } else { + calciteColLst.add(originalInputRefs.get(i)); + } + } + outputRel = genSelectRelNode(calciteColLst, out_rwsch, srcRel); + // outputRel is the generated augmented select with extra unselected + // columns, and originalRR is the original generated select + return new Pair(outputRel, originalRR); + } else { + outputRel = genSelectRelNode(calciteColLst, out_rwsch, srcRel); + } } - // 9. Handle select distinct as GBY if there exist windowing functions if (selForWindow != null && selExprList.getToken().getType() == HiveParser.TOK_SELECTDI) { ImmutableBitSet groupSet = ImmutableBitSet.range(outputRel.getRowType().getFieldList().size()); @@ -3724,7 +3794,7 @@ private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel) this.relToHiveRR.put(outputRel, groupByOutputRowResolver); } - return outputRel; + return new Pair(outputRel, null); } private RelNode genUDTFPlan(GenericUDTF genericUDTF, String genericUDTFName, String outputTableAlias, @@ -3946,11 +4016,12 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB, srcRel = (gbHavingRel == null) ? srcRel : gbHavingRel; // 5. Build Rel for Select Clause - selectRel = genSelectLogicalPlan(qb, srcRel, starSrcRel); + Pair selPair = genSelectLogicalPlan(qb, srcRel, starSrcRel, true); + selectRel = selPair.getKey(); srcRel = (selectRel == null) ? srcRel : selectRel; // 6. Build Rel for OB Clause - Pair obTopProjPair = genOBLogicalPlan(qb, srcRel, outerMostQB); + Pair obTopProjPair = genOBLogicalPlan(qb, selPair, outerMostQB); obRel = obTopProjPair.getKey(); RelNode topConstrainingProjArgsRel = obTopProjPair.getValue(); srcRel = (obRel == null) ? srcRel : obRel; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java index e14f1cf..5e739d5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java @@ -178,23 +178,31 @@ public ColumnInfo get(String tab_alias, String col_alias) throws SemanticExcepti } ret = f_map.get(col_alias); } else { - boolean found = false; - String foundTbl = null; - for (Map.Entry> rslvEntry: rslvMap.entrySet()) { - String rslvKey = rslvEntry.getKey(); - LinkedHashMap cmap = rslvEntry.getValue(); - for (Map.Entry cmapEnt : cmap.entrySet()) { - if (col_alias.equalsIgnoreCase(cmapEnt.getKey())) { - /* - * We can have an unaliased and one aliased mapping to a Column. - */ - if (found && foundTbl != null && rslvKey != null) { - throw new SemanticException("Column " + col_alias - + " Found in more than One Tables/Subqueries"); + HashMap f_map = rslvMap.get(tab_alias); + // first try rslvMap directly + if (f_map != null) { + ret = f_map.get(col_alias); + } + // then try others + if (ret == null) { + boolean found = false; + String foundTbl = null; + for (Map.Entry> rslvEntry : rslvMap.entrySet()) { + String rslvKey = rslvEntry.getKey(); + LinkedHashMap cmap = rslvEntry.getValue(); + for (Map.Entry cmapEnt : cmap.entrySet()) { + if (col_alias.equalsIgnoreCase(cmapEnt.getKey())) { + /* + * We can have an unaliased and one aliased mapping to a Column. + */ + if (found && foundTbl != null && rslvKey != null) { + throw new SemanticException("Column " + col_alias + + " Found in more than One Tables/Subqueries"); + } + found = true; + foundTbl = rslvKey == null ? foundTbl : rslvKey; + ret = cmapEnt.getValue(); } - found = true; - foundTbl = rslvKey == null ? foundTbl : rslvKey; - ret = cmapEnt.getValue(); } } } @@ -458,7 +466,11 @@ public static RowResolver getCombinedRR(RowResolver leftRR, public RowResolver duplicate() { RowResolver resolver = new RowResolver(); resolver.rowSchema = new RowSchema(rowSchema); - resolver.rslvMap.putAll(rslvMap); + for (Map.Entry> entry : rslvMap.entrySet()) { + LinkedHashMap map = new LinkedHashMap<>(); + map.putAll(entry.getValue()); + resolver.rslvMap.put(entry.getKey(), map); + } resolver.invRslvMap.putAll(invRslvMap); resolver.altInvRslvMap.putAll(altInvRslvMap); resolver.expressionMap.putAll(expressionMap); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0732207..42f1f86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1613,6 +1613,8 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan throw new SemanticException(generateErrorMessage(ast, ErrorMsg.CLUSTERBY_ORDERBY_CONFLICT.getMsg())); } + qbp.addAggregationExprsForClause(ctx_1.dest, + doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest)); break; case HiveParser.TOK_GROUPBY: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index f979c14..3b2f790 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -163,7 +163,7 @@ public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) if (colInfo != null) { desc = new ExprNodeColumnDesc(colInfo); ASTNode source = input.getExpressionSource(expr); - if (source != null) { + if (source != null && ctx.getUnparseTranslator() != null) { ctx.getUnparseTranslator().addCopyTranslation(expr, source); } return desc; diff --git a/ql/src/test/queries/clientpositive/order_by_expr_1.q b/ql/src/test/queries/clientpositive/order_by_expr_1.q new file mode 100644 index 0000000..1d99e6a --- /dev/null +++ b/ql/src/test/queries/clientpositive/order_by_expr_1.q @@ -0,0 +1,44 @@ +set hive.fetch.task.conversion=none; + +create table t(a int, b int); + +insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9); + +select a, count(a) from t group by a order by count(a), a; + +explain +select + interval '2-2' year to month + interval '3-3' year to month, + interval '2-2' year to month - interval '3-3' year to month +from t +order by interval '2-2' year to month + interval '3-3' year to month +limit 2; + +select a,b, count(*) from t group by a, b order by a+b; +select a,b, count(*) from t group by a, b order by count(*), b desc; +select a,b,count(*),a+b from t group by a, b order by a+b; +select a,b from t order by a+b; +select a,b,a+b from t order by a+b; +select a,b,a+b from t order by a+b desc; +select cast(0.99999999999999999999 as decimal(20,19)) as c from t limit 1; +select cast(0.99999999999999999999 as decimal(20,19)) as c from t order by c limit 1; +select a from t order by b; +select a from t order by 0-b; +select b from t order by 0-b; +select b from t order by a, 0-b; +select b from t order by a+1, 0-b; +select b from t order by 0-b, a+1; +explain select b from t order by 0-b, a+1; +select a,b from t order by 0-b; +select a,b from t order by a, a+1, 0-b; +select a,b from t order by 0-b, a+1; +select a+1,b from t order by a, a+1, 0-b; +select a+1 as c, b from t order by a, a+1, 0-b; +select a, a+1 as c, b from t order by a, a+1, 0-b; +select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b; +explain select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b; +select a, a+1 as c, b, 2*b from t order by a+1, 0-b; +select a,b, count(*) as c from t group by a, b order by c, a+b desc; + +select a, max(b) from t group by a order by count(b), a desc; +select a, max(b) from t group by a order by count(b), a; diff --git a/ql/src/test/queries/clientpositive/order_by_expr_2.q b/ql/src/test/queries/clientpositive/order_by_expr_2.q new file mode 100644 index 0000000..043f8ed --- /dev/null +++ b/ql/src/test/queries/clientpositive/order_by_expr_2.q @@ -0,0 +1,11 @@ +set hive.fetch.task.conversion=none; + +create table t(a int, b int); + +insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9); + +select a as b, b as a from t order by a; +select a as b, b as a from t order by t.a; +select a as b from t order by b; +select a as b from t order by 0-a; +select a,b,count(*),a+b from t group by a, b order by a+b; diff --git a/ql/src/test/results/clientpositive/order_by_expr_1.q.out b/ql/src/test/results/clientpositive/order_by_expr_1.q.out new file mode 100644 index 0000000..39babb7 --- /dev/null +++ b/ql/src/test/results/clientpositive/order_by_expr_1.q.out @@ -0,0 +1,566 @@ +PREHOOK: query: create table t(a int, b int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t(a int, b int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9) +PREHOOK: type: QUERY +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: t.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select a, count(a) from t group by a order by count(a), a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, count(a) from t group by a order by count(a), a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 1 +2 1 +3 1 +4 1 +8 1 +20 1 +1 3 +PREHOOK: query: explain +select + interval '2-2' year to month + interval '3-3' year to month, + interval '2-2' year to month - interval '3-3' year to month +from t +order by interval '2-2' year to month + interval '3-3' year to month +limit 2 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + interval '2-2' year to month + interval '3-3' year to month, + interval '2-2' year to month - interval '3-3' year to month +from t +order by interval '2-2' year to month + interval '3-3' year to month +limit 2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9 Data size: 144 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 2 + Processor Tree: + ListSink + +PREHOOK: query: select a,b, count(*) from t group by a, b order by a+b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b, count(*) from t group by a, b order by a+b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 1 +20 -100 1 +1 2 2 +1 3 1 +2 4 1 +4 5 1 +3 7 1 +8 9 1 +PREHOOK: query: select a,b, count(*) from t group by a, b order by count(*), b desc +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b, count(*) from t group by a, b order by count(*), b desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 1 +8 9 1 +3 7 1 +4 5 1 +2 4 1 +1 3 1 +20 -100 1 +1 2 2 +PREHOOK: query: select a,b,count(*),a+b from t group by a, b order by a+b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b,count(*),a+b from t group by a, b order by a+b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 1 -900 +20 -100 1 -80 +1 2 2 3 +1 3 1 4 +2 4 1 6 +4 5 1 9 +3 7 1 10 +8 9 1 17 +PREHOOK: query: select a,b from t order by a+b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b from t order by a+b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +20 -100 +1 2 +1 2 +1 3 +2 4 +4 5 +3 7 +8 9 +PREHOOK: query: select a,b,a+b from t order by a+b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b,a+b from t order by a+b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 -900 +20 -100 -80 +1 2 3 +1 2 3 +1 3 4 +2 4 6 +4 5 9 +3 7 10 +8 9 17 +PREHOOK: query: select a,b,a+b from t order by a+b desc +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b,a+b from t order by a+b desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +8 9 17 +3 7 10 +4 5 9 +2 4 6 +1 3 4 +1 2 3 +1 2 3 +20 -100 -80 +-1000 100 -900 +PREHOOK: query: select cast(0.99999999999999999999 as decimal(20,19)) as c from t limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select cast(0.99999999999999999999 as decimal(20,19)) as c from t limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1.0000000000000000000 +PREHOOK: query: select cast(0.99999999999999999999 as decimal(20,19)) as c from t order by c limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select cast(0.99999999999999999999 as decimal(20,19)) as c from t order by c limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1.0000000000000000000 +PREHOOK: query: select a from t order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a from t order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +20 +1 +1 +1 +2 +4 +3 +8 +-1000 +PREHOOK: query: select a from t order by 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a from t order by 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 +8 +3 +4 +2 +1 +1 +1 +20 +PREHOOK: query: select b from t order by 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select b from t order by 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +100 +9 +7 +5 +4 +3 +2 +2 +-100 +PREHOOK: query: select b from t order by a, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select b from t order by a, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +100 +3 +2 +2 +4 +7 +5 +9 +-100 +PREHOOK: query: select b from t order by a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select b from t order by a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +100 +3 +2 +2 +4 +7 +5 +9 +-100 +PREHOOK: query: select b from t order by 0-b, a+1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select b from t order by 0-b, a+1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +100 +9 +7 +5 +4 +3 +2 +2 +-100 +PREHOOK: query: explain select b from t order by 0-b, a+1 +PREHOOK: type: QUERY +POSTHOOK: query: explain select b from t order by 0-b, a+1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: b (type: int), (0 - b) (type: int), (a + 1) (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), _col2 (type: int) + sort order: ++ + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a,b from t order by 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b from t order by 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +8 9 +3 7 +4 5 +2 4 +1 3 +1 2 +1 2 +20 -100 +PREHOOK: query: select a,b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +1 3 +1 2 +1 2 +2 4 +3 7 +4 5 +8 9 +20 -100 +PREHOOK: query: select a,b from t order by 0-b, a+1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b from t order by 0-b, a+1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +8 9 +3 7 +4 5 +2 4 +1 3 +1 2 +1 2 +20 -100 +PREHOOK: query: select a+1,b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a+1,b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-999 100 +2 3 +2 2 +2 2 +3 4 +4 7 +5 5 +9 9 +21 -100 +PREHOOK: query: select a+1 as c, b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a+1 as c, b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-999 100 +2 3 +2 2 +2 2 +3 4 +4 7 +5 5 +9 9 +21 -100 +PREHOOK: query: select a, a+1 as c, b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, a+1 as c, b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 -999 100 +1 2 3 +1 2 2 +1 2 2 +2 3 4 +3 4 7 +4 5 5 +8 9 9 +20 21 -100 +PREHOOK: query: select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 -999 100 200 +1 2 3 6 +1 2 2 4 +1 2 2 4 +2 3 4 8 +3 4 7 14 +4 5 5 10 +8 9 9 18 +20 21 -100 -200 +PREHOOK: query: explain select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b +PREHOOK: type: QUERY +POSTHOOK: query: explain select a, a+1 as c, b, 2*b from t order by a, a+1, 0-b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: int), (2 * b) (type: int), (a + 1) (type: int), (0 - b) (type: int) + outputColumnNames: _col0, _col2, _col3, _col4, _col5 + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col4 (type: int), _col5 (type: int) + sort order: +++ + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: int), _col3 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 37 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select a, a+1 as c, b, 2*b from t order by a+1, 0-b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, a+1 as c, b, 2*b from t order by a+1, 0-b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 -999 100 200 +1 2 3 6 +1 2 2 4 +1 2 2 4 +2 3 4 8 +3 4 7 14 +4 5 5 10 +8 9 9 18 +20 21 -100 -200 +PREHOOK: query: select a,b, count(*) as c from t group by a, b order by c, a+b desc +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b, count(*) as c from t group by a, b order by c, a+b desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +8 9 1 +3 7 1 +4 5 1 +2 4 1 +1 3 1 +20 -100 1 +-1000 100 1 +1 2 2 +PREHOOK: query: select a, max(b) from t group by a order by count(b), a desc +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, max(b) from t group by a order by count(b), a desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +20 -100 +8 9 +4 5 +3 7 +2 4 +-1000 100 +1 3 +PREHOOK: query: select a, max(b) from t group by a order by count(b), a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a, max(b) from t group by a order by count(b), a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +2 4 +3 7 +4 5 +8 9 +20 -100 +1 3 diff --git a/ql/src/test/results/clientpositive/order_by_expr_2.q.out b/ql/src/test/results/clientpositive/order_by_expr_2.q.out new file mode 100644 index 0000000..4b835b0 --- /dev/null +++ b/ql/src/test/results/clientpositive/order_by_expr_2.q.out @@ -0,0 +1,100 @@ +PREHOOK: query: create table t(a int, b int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t(a int, b int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9) +PREHOOK: type: QUERY +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: t.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select a as b, b as a from t order by a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a as b, b as a from t order by a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +20 -100 +1 2 +1 2 +1 3 +2 4 +4 5 +3 7 +8 9 +-1000 100 +PREHOOK: query: select a as b, b as a from t order by t.a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a as b, b as a from t order by t.a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 +1 3 +1 2 +1 2 +2 4 +3 7 +4 5 +8 9 +20 -100 +PREHOOK: query: select a as b from t order by b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a as b from t order by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 +1 +1 +1 +2 +3 +4 +8 +20 +PREHOOK: query: select a as b from t order by 0-a +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a as b from t order by 0-a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +20 +8 +4 +3 +2 +1 +1 +1 +-1000 +PREHOOK: query: select a,b,count(*),a+b from t group by a, b order by a+b +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select a,b,count(*),a+b from t group by a, b order by a+b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +-1000 100 1 -900 +20 -100 1 -80 +1 2 2 3 +1 3 1 4 +2 4 1 6 +4 5 1 9 +3 7 1 10 +8 9 1 17