diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index e254505..2ddcecb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -116,6 +116,7 @@ import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.eigenbase.rel.RelNode; public class Driver implements CommandProcessor { @@ -304,6 +305,41 @@ public int compile(String command) { } /** + * Generate Hive Optiq logical plan from a given hive sql command + * @param command + * @return Hive optiq RelNode + * @throws Exception + */ + public RelNode getOptiqLogicalPlan(String command) throws Exception { + SessionState ss = new SessionState(new HiveConf(SessionState.class)); + SessionState.start(ss); + + QueryState queryState = new QueryState(); + saveSession(queryState); + + Context ctx = new Context(conf); + + SessionState.get().initTxnMgr(conf); + + ParseDriver pd = new ParseDriver(); + ASTNode tree = pd.parse(command, ctx); + tree = ParseUtils.findRootNonNullToken(tree); + + BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree); + RelNode relNode = null; + if (sem instanceof SemanticAnalyzer) { + relNode = ((SemanticAnalyzer) sem).genOptiqLogicalPlan(tree, ctx); + } else { + return null; + } + + restoreSession(queryState); + ss.close(); + + return relNode; + } + + /** * Hold state variables specific to each query being executed, that may not * be consistent in the overall SessionState */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java index f8755d0..6bc1a3b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveLimitRel.java @@ -30,8 +30,8 @@ import org.eigenbase.rex.RexNode; public class HiveLimitRel extends SingleRel implements HiveRel { - private final RexNode offset; - private final RexNode fetch; + public final RexNode offset; + public final RexNode fetch; HiveLimitRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RexNode offset, RexNode fetch) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 511103b..c7d7a6f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -9847,6 +9848,56 @@ public void init(boolean clearPartsCache) { this.qb = qb; } + /** + * generate Optiq operator tree for given ASTNode + * + * @param hive AST tree root node + * @param hive execution Context + */ + public RelNode genOptiqLogicalPlan(ASTNode ast, Context ctx) throws SemanticException{ + + initCtx(ctx); + init(true); + + ASTNode child = ast; + this.ast = ast; + viewsExpanded = new ArrayList(); + ctesExpanded = new ArrayList(); + LOG.info("Starting Semantic Analysis"); + + // analyze and process the position alias + processPositionAlias(ast); + + boolean tokenTypesQuery = ast.getToken().getType() == HiveParser.TOK_QUERY; + if (!tokenTypesQuery ) { + return null; + } + + // continue analyzing from the child ASTNode. + Phase1Ctx ctx_1 = initPhase1Ctx(); + if (!doPhase1(child, qb, ctx_1)) { + // if phase1Result false return + return null; + } + + LOG.info("Completed phase 1 of Semantic Analysis"); + + if (queryProperties.hasClusterBy() || queryProperties.hasDistributeBy() + || queryProperties.hasSortBy() || queryProperties.hasPTF() + || queryProperties.usesScript() || queryProperties.hasMultiDestQuery() + || queryProperties.hasLateralViews()) { + LOG.info("Can not invoke CBO; query contains operators not supported for CBO."); + return null; + } + + getMetaData(qb); + LOG.info("Completed getting MetaData in Semantic Analysis"); + + OptiqPlanner optiqPlanner = new OptiqPlanner(); + RelNode result = optiqPlanner.getOptiqPlan(prunedPartitions); + return result; + } + @Override @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { @@ -12254,12 +12305,47 @@ private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) { return runOptiqPlanner; } + public class OptiqPlanner extends OptiqBasedPlanner { + @Override + public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) { + RelNode optiqGenPlan = null; + + /* + * recreate cluster, so that it picks up the additional traitDef + */ + RelOptPlanner planner = HiveVolcanoPlanner.createPlanner(); + final RelOptQuery query = new RelOptQuery(planner); + final RexBuilder rexBuilder = cluster.getRexBuilder(); + cluster = query.createCluster(rexBuilder.getTypeFactory(), rexBuilder); + + this.cluster = cluster; + try { + optiqGenPlan = genLogicalPlan(qb, true); + } catch (SemanticException e) { + throw new RuntimeException(e); + } + return optiqGenPlan; + } + + public RelNode getOptiqPlan(Map partitionCache) { + this.partitionCache = partitionCache; + RelNode optiqPlan = null; + try { + optiqPlan = Frameworks.withPlanner(this, + Frameworks.newConfigBuilder().typeSystem(new HiveTypeSystemImpl()).build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + return optiqPlan; + } + } + private class OptiqBasedPlanner implements Frameworks.PlannerAction { - private RelOptCluster cluster; - private RelOptSchema relOptSchema; - private SemanticException semanticException; - private Map partitionCache; - private final AtomicInteger noColsMissingStats = new AtomicInteger(0); + protected RelOptCluster cluster; + protected RelOptSchema relOptSchema; + protected SemanticException semanticException; + protected Map partitionCache; + protected final AtomicInteger noColsMissingStats = new AtomicInteger(0); List topLevelFieldSchema; // TODO: Do we need to keep track of RR, ColNameToPosMap for every op or @@ -14020,7 +14106,7 @@ private RelNode genLogicalPlan(QBExpr qbexpr) throws SemanticException { return null; } - private RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticException { + protected RelNode genLogicalPlan(QB qb, boolean outerMostQB) throws SemanticException { RelNode srcRel = null; RelNode filterRel = null; RelNode gbRel = null;