diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java index 2f309f3..ecbffc8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexUtil.java @@ -315,6 +315,12 @@ private static RexCall negate(RexBuilder rexBuilder, RexCall call) { private static RexCall invert(RexBuilder rexBuilder, RexCall call) { switch (call.getKind()) { + case EQUALS: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + Lists.reverse(call.getOperands())); + case NOT_EQUALS: + return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS, + Lists.reverse(call.getOperands())); case LESS_THAN: return (RexCall) rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, Lists.reverse(call.getOperands())); @@ -330,4 +336,22 @@ private static RexCall invert(RexBuilder rexBuilder, RexCall call) { } return null; } + + public static SqlKind invert(SqlKind kind) { + switch (kind) { + case EQUALS: + return SqlKind.EQUALS; + case NOT_EQUALS: + return SqlKind.NOT_EQUALS; + case LESS_THAN: + return SqlKind.GREATER_THAN; + case GREATER_THAN: + return SqlKind.LESS_THAN; + case LESS_THAN_OR_EQUAL: + return SqlKind.GREATER_THAN_OR_EQUAL; + case GREATER_THAN_OR_EQUAL: + return SqlKind.LESS_THAN_OR_EQUAL; + } + return null; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java new file mode 100644 index 0000000..cf1f5c8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules; + +import java.math.BigDecimal; +import java.util.EnumSet; +import java.util.Set; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.metadata.RelColumnOrigin; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil; +import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.apache.hadoop.hive.ql.plan.ColStatistics; +import org.apache.hadoop.hive.ql.plan.ColStatistics.Range; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class HiveReduceExpressionsWithStatsRule extends RelOptRule { + + protected static final Logger LOG = LoggerFactory.getLogger( + HiveReduceExpressionsWithStatsRule.class); + + public static final HiveReduceExpressionsWithStatsRule INSTANCE = + new HiveReduceExpressionsWithStatsRule(); + + private static final Set COMPARISON = EnumSet.of(SqlKind.EQUALS, + SqlKind.GREATER_THAN_OR_EQUAL, + SqlKind.LESS_THAN_OR_EQUAL, + SqlKind.GREATER_THAN, + SqlKind.LESS_THAN, + SqlKind.NOT_EQUALS); + + private HiveReduceExpressionsWithStatsRule() { + super(operand(Filter.class, operand(RelNode.class, any()))); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Filter filter = call.rel(0); + + final RexBuilder rexBuilder = filter.getCluster().getRexBuilder(); + final RelMetadataQuery metadataProvider = RelMetadataQuery.instance(); + + // 1. Recompose filter possibly by pulling out common elements from DNF + // expressions + RexNode newFilterCondition = RexUtil.pullFactors(rexBuilder, filter.getCondition()); + + // 2. Reduce filter with stats information + RexReplacer replacer = new RexReplacer(filter, rexBuilder, metadataProvider); + newFilterCondition = replacer.apply(newFilterCondition); + + // 3. Transform if we have created a new filter operator + if (!filter.getCondition().toString().equals(newFilterCondition.toString())) { + Filter newFilter = filter.copy(filter.getTraitSet(), filter.getInput(), newFilterCondition); + call.transformTo(newFilter); + } + + } + + /** + * Replaces expressions with their reductions. Note that we only have to + * look for RexCall, since nothing else is reducible in the first place. + */ + protected static class RexReplacer extends RexShuttle { + private final Filter filterOp; + private final RexBuilder rexBuilder; + private final RelMetadataQuery metadataProvider; + + RexReplacer(Filter filterOp, RexBuilder rexBuilder, RelMetadataQuery metadataProvider) { + this.filterOp = filterOp; + this.rexBuilder = rexBuilder; + this.metadataProvider = metadataProvider; + } + + @SuppressWarnings("unchecked") + @Override public RexNode visitCall(RexCall call) { + if (COMPARISON.contains(call.getOperator().getKind())) { + RexInputRef ref = null; + RexLiteral literal = null; + SqlKind kind = null; + if (call.operands.get(0) instanceof RexInputRef + && call.operands.get(1) instanceof RexLiteral) { + ref = (RexInputRef) call.operands.get(0); + literal = (RexLiteral) call.operands.get(1); + kind = call.getOperator().getKind(); + } else if (call.operands.get(1) instanceof RexInputRef + && call.operands.get(0) instanceof RexLiteral) { + ref = (RexInputRef) call.operands.get(1); + literal = (RexLiteral) call.operands.get(0); + kind = HiveRexUtil.invert(call.getOperator().getKind()); + } + + // Found an expression that we can try to reduce + Number max = null; + Number min = null; + if (ref != null && literal != null && kind != null) { + RelColumnOrigin columnOrigin = this.metadataProvider.getColumnOrigin(filterOp, ref.getIndex()); + if (columnOrigin != null) { + RelOptHiveTable table = (RelOptHiveTable) columnOrigin.getOriginTable(); + if (table != null) { + ColStatistics colStats = + table.getColStat(Lists.newArrayList(columnOrigin.getOriginColumnOrdinal())).get(0); + if (colStats != null) { + Range range = colStats.getRange(); + if (range != null) { + max = range.maxValue; + min = range.minValue; + } + } + } + } + } + + // Stats were available, try to reduce + if (max != null && min != null) { + BigDecimal maxVal = new BigDecimal(max.floatValue()); + BigDecimal minVal = new BigDecimal(min.floatValue()); + RexLiteral maxLiteral = rexBuilder.makeExactLiteral(maxVal, literal.getType()); + RexLiteral minLiteral = rexBuilder.makeExactLiteral(minVal, literal.getType()); + + // Equals + if (kind == SqlKind.EQUALS) { + if (minLiteral.getValue().compareTo(literal.getValue()) > 0 || + maxLiteral.getValue().compareTo(literal.getValue()) < 0) { + return rexBuilder.makeLiteral(false); + } + } + + // Greater than (or equal), and less than (or equal) + if (kind == SqlKind.GREATER_THAN) { + if (minLiteral.getValue().compareTo(literal.getValue()) > 0) { + return rexBuilder.makeLiteral(true); + } else if (maxLiteral.getValue().compareTo(literal.getValue()) <= 0) { + return rexBuilder.makeLiteral(false); + } + } else if (kind == SqlKind.GREATER_THAN_OR_EQUAL) { + if (minLiteral.getValue().compareTo(literal.getValue()) >= 0) { + return rexBuilder.makeLiteral(true); + } else if (maxLiteral.getValue().compareTo(literal.getValue()) < 0) { + return rexBuilder.makeLiteral(false); + } + } else if (kind == SqlKind.LESS_THAN) { + if (minLiteral.getValue().compareTo(literal.getValue()) >= 0) { + return rexBuilder.makeLiteral(false); + } else if (maxLiteral.getValue().compareTo(literal.getValue()) < 0) { + return rexBuilder.makeLiteral(true); + } + } else { + if (minLiteral.getValue().compareTo(literal.getValue()) > 0) { + return rexBuilder.makeLiteral(false); + } else if (maxLiteral.getValue().compareTo(literal.getValue()) <= 0) { + return rexBuilder.makeLiteral(true); + } + } + } + + // We cannot apply the reduction + return call; + } + + // If we did not reduce, check the children nodes + RexNode node = super.visitCall(call); + if (node != call) { + node = HiveRexUtil.simplify(rexBuilder, node); + } + return node; + } + + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 88ed0db..9762793 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -159,6 +159,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsWithStatsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule; @@ -1150,6 +1151,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule(HiveRelFactories.HIVE_FILTER_FACTORY), + HiveReduceExpressionsWithStatsRule.INSTANCE, HiveReduceExpressionsRule.PROJECT_INSTANCE, HiveReduceExpressionsRule.FILTER_INSTANCE, HiveReduceExpressionsRule.JOIN_INSTANCE, diff --git ql/src/test/queries/clientpositive/remove_exprs_stats.q ql/src/test/queries/clientpositive/remove_exprs_stats.q new file mode 100644 index 0000000..0fdf11d --- /dev/null +++ ql/src/test/queries/clientpositive/remove_exprs_stats.q @@ -0,0 +1,45 @@ +set hive.mapred.mode=nonstrict; +set hive.stats.fetch.column.stats=true; + +create table if not exists loc_staging ( + state string, + locid int, + zip bigint, + year int +) row format delimited fields terminated by '|' stored as textfile; + +create table loc_orc like loc_staging; +alter table loc_orc set fileformat orc; + +load data local inpath '../../data/files/loc.txt' overwrite into table loc_staging; + +insert overwrite table loc_orc select * from loc_staging; + +analyze table loc_orc compute statistics for columns state,locid,zip,year; + +-- always true +explain select * from loc_orc where locid < 30; +-- always false +explain select * from loc_orc where locid > 30; +-- always true +explain select * from loc_orc where locid <= 30; +-- always false +explain select * from loc_orc where locid >= 30; + +-- nothing to do +explain select * from loc_orc where locid < 6; +-- always false +explain select * from loc_orc where locid > 6; +-- always true +explain select * from loc_orc where locid <= 6; +-- nothing to do +explain select * from loc_orc where locid >= 6; + +-- always false +explain select * from loc_orc where locid < 1; +-- nothing to do +explain select * from loc_orc where locid > 1; +-- nothing to do +explain select * from loc_orc where locid <= 1; +-- always true +explain select * from loc_orc where locid >= 1; diff --git ql/src/test/results/clientpositive/remove_exprs_stats.q.out ql/src/test/results/clientpositive/remove_exprs_stats.q.out new file mode 100644 index 0000000..2205e07 --- /dev/null +++ ql/src/test/results/clientpositive/remove_exprs_stats.q.out @@ -0,0 +1,458 @@ +PREHOOK: query: create table if not exists loc_staging ( + state string, + locid int, + zip bigint, + year int +) row format delimited fields terminated by '|' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@loc_staging +POSTHOOK: query: create table if not exists loc_staging ( + state string, + locid int, + zip bigint, + year int +) row format delimited fields terminated by '|' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@loc_staging +PREHOOK: query: create table loc_orc like loc_staging +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@loc_orc +POSTHOOK: query: create table loc_orc like loc_staging +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@loc_orc +PREHOOK: query: alter table loc_orc set fileformat orc +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc +POSTHOOK: query: alter table loc_orc set fileformat orc +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc +PREHOOK: query: load data local inpath '../../data/files/loc.txt' overwrite into table loc_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@loc_staging +POSTHOOK: query: load data local inpath '../../data/files/loc.txt' overwrite into table loc_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@loc_staging +PREHOOK: query: insert overwrite table loc_orc select * from loc_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@loc_staging +PREHOOK: Output: default@loc_orc +POSTHOOK: query: insert overwrite table loc_orc select * from loc_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@loc_staging +POSTHOOK: Output: default@loc_orc +POSTHOOK: Lineage: loc_orc.locid SIMPLE [(loc_staging)loc_staging.FieldSchema(name:locid, type:int, comment:null), ] +POSTHOOK: Lineage: loc_orc.state SIMPLE [(loc_staging)loc_staging.FieldSchema(name:state, type:string, comment:null), ] +POSTHOOK: Lineage: loc_orc.year SIMPLE [(loc_staging)loc_staging.FieldSchema(name:year, type:int, comment:null), ] +POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name:zip, type:bigint, comment:null), ] +PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year +PREHOOK: type: QUERY +PREHOOK: Input: default@loc_orc +#### A masked pattern was here #### +POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year +POSTHOOK: type: QUERY +POSTHOOK: Input: default@loc_orc +#### A masked pattern was here #### +PREHOOK: query: -- always true +explain select * from loc_orc where locid < 30 +PREHOOK: type: QUERY +POSTHOOK: query: -- always true +explain select * from loc_orc where locid < 30 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: -- always false +explain select * from loc_orc where locid > 30 +PREHOOK: type: QUERY +POSTHOOK: query: -- always false +explain select * from loc_orc where locid > 30 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- always true +explain select * from loc_orc where locid <= 30 +PREHOOK: type: QUERY +POSTHOOK: query: -- always true +explain select * from loc_orc where locid <= 30 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: -- always false +explain select * from loc_orc where locid >= 30 +PREHOOK: type: QUERY +POSTHOOK: query: -- always false +explain select * from loc_orc where locid >= 30 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- nothing to do +explain select * from loc_orc where locid < 6 +PREHOOK: type: QUERY +POSTHOOK: query: -- nothing to do +explain select * from loc_orc where locid < 6 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid < 6) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- always false +explain select * from loc_orc where locid > 6 +PREHOOK: type: QUERY +POSTHOOK: query: -- always false +explain select * from loc_orc where locid > 6 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- always true +explain select * from loc_orc where locid <= 6 +PREHOOK: type: QUERY +POSTHOOK: query: -- always true +explain select * from loc_orc where locid <= 6 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: -- nothing to do +explain select * from loc_orc where locid >= 6 +PREHOOK: type: QUERY +POSTHOOK: query: -- nothing to do +explain select * from loc_orc where locid >= 6 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid >= 6) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- always false +explain select * from loc_orc where locid < 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- always false +explain select * from loc_orc where locid < 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- nothing to do +explain select * from loc_orc where locid > 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- nothing to do +explain select * from loc_orc where locid > 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid > 1) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- nothing to do +explain select * from loc_orc where locid <= 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- nothing to do +explain select * from loc_orc where locid <= 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid <= 1) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- always true +explain select * from loc_orc where locid >= 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- always true +explain select * from loc_orc where locid >= 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + ListSink +