diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java index d1b2a72..f2ebf6e 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java @@ -320,9 +320,23 @@ private static String createSelectStarQuery(String dataSource) throws IOExceptio return splits; } + /** + * Extract the total time span covered by these intervals. It does not check + * if the intervals overlap. + * @param intervals list of intervals + * @return total time span covered by these intervals + */ + private static long extractTotalTime(List intervals) { + long totalTime = 0; + for (Interval interval : intervals) { + totalTime += interval.getEndMillis() - interval.getStartMillis(); + } + return totalTime; + } + private static List> createSplitsIntervals(List intervals, int numSplits ) { - final long totalTime = DruidDateTimeUtils.extractTotalTime(intervals); + final long totalTime = extractTotalTime(intervals); long startTime = intervals.get(0).getStartMillis(); long endTime = startTime; long currTime = 0; diff --git a/pom.xml b/pom.xml index c2e9ac3..c19f4be 100644 --- a/pom.xml +++ b/pom.xml @@ -113,10 +113,10 @@ 3.5.2 1.5.6 0.1 - 1.8.0 + 1.9.0 1.7.7 0.8.0.RELEASE - 1.10.0 + 1.12.0-SNAPSHOT 4.2.1 4.1.6 4.1.7 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java index f1e8ebd..43e9f5e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSubQueryRemoveRule.java @@ -277,10 +277,17 @@ protected RexNode apply(RexSubQuery e, Set variablesSet, b = null; // fall through case UNKNOWN_AS_TRUE: - operands.add( + RexNode bLiteral = null; + if(b == null){ + bLiteral = e.rel.getCluster().getRexBuilder().makeNullLiteral(SqlTypeName.BOOLEAN); + } + else { + bLiteral = builder.literal(b); + } + operands.add( builder.call(SqlStdOperatorTable.LESS_THAN, builder.field("ct", "ck"), builder.field("ct", "c")), - builder.literal(b)); + bLiteral); break; } operands.add(builder.literal(false)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index 69e157e..5fae15e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -165,7 +165,7 @@ public RelOptPredicateList getPredicates(Project project, RelMetadataQuery mq) { rexBuilder.makeInputRef(project, expr.i), expr.e)); } } - return RelOptPredicateList.of(projectPullUpPredicates); + return RelOptPredicateList.of(rexBuilder, projectPullUpPredicates); } /** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */ @@ -183,7 +183,7 @@ public RelOptPredicateList getPredicates(Join join, RelMetadataQuery mq) { RexUtil.composeConjunction(rB, rightInfo.pulledUpPredicates, false)); - return jI.inferPredicates(false); + return jI.inferPredicates(false, rB); } /** @@ -219,7 +219,7 @@ public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq) { aggPullUpPredicates.add(r); } } - return RelOptPredicateList.of(aggPullUpPredicates); + return RelOptPredicateList.of(agg.getCluster().getRexBuilder(), aggPullUpPredicates); } /** @@ -271,7 +271,7 @@ public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) { if (!disjPred.isAlwaysTrue()) { preds.add(disjPred); } - return RelOptPredicateList.of(preds); + return RelOptPredicateList.of(rB, preds); } /** @@ -404,7 +404,8 @@ public Void apply(RexNode input) { * */ public RelOptPredicateList inferPredicates( - boolean includeEqualityInference) { + boolean includeEqualityInference, + RexBuilder rB) { final List inferredPredicates = new ArrayList<>(); final List nonFieldsPredicates = new ArrayList<>(); final Set allExprsDigests = new HashSet<>(this.allExprsDigests); @@ -476,13 +477,13 @@ public RelOptPredicateList inferPredicates( pulledUpPredicates = Iterables.concat(leftPreds, rightPreds, RelOptUtil.conjunctions(joinRel.getCondition()), inferredPredicates); } - return RelOptPredicateList.of( + return RelOptPredicateList.of(rB, pulledUpPredicates, leftInferredPredicates, rightInferredPredicates); case LEFT: - return RelOptPredicateList.of( + return RelOptPredicateList.of(rB, leftPreds, EMPTY_LIST, rightInferredPredicates); case RIGHT: - return RelOptPredicateList.of( + return RelOptPredicateList.of(rB, rightPreds, leftInferredPredicates, EMPTY_LIST); default: assert inferredPredicates.size() == 0; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 9f1b9d5..98b5ec0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2276,7 +2276,7 @@ private void subqueryRestrictionCheck(QB qb, ASTNode searchCond, RelNode srcRel, ASTNode subQueryAST = subQueries.get(i); - SubQueryUtils.rewriteParentQueryWhere(clonedSearchCond, subQueryAST); + //SubQueryUtils.rewriteParentQueryWhere(clonedSearchCond, subQueryAST); QBSubQuery subQuery = SubQueryUtils.buildSubQuery(qb.getId(), sqIdx, subQueryAST, originalSubQueryAST, ctx);