diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index cfea602fd8..0475264db8 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -788,8 +788,8 @@ public int run(String[] args) throws Exception {
ss.updateThreadName();
- // Initialize metadata provider class
- CalcitePlanner.initializeMetadataProviderClass();
+ // Initialize metadata provider class and trimmer
+ CalcitePlanner.warmup();
// Create views registry
HiveMaterializedViewsRegistry.get().init();
diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/calcite/FieldTrimmerBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/calcite/FieldTrimmerBench.java
new file mode 100644
index 0000000000..d98e2511a1
--- /dev/null
+++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/calcite/FieldTrimmerBench.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.benchmark.calcite;
+
+import com.google.common.collect.Lists;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.AbstractRelNode;
+import org.apache.calcite.rel.BiRel;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.SingleRel;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelRecordType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer;
+import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+/**
+ * This test measures the performance for field trimmer.
+ *
+ * This test uses JMH framework for benchmarking.
+ * You may execute this benchmark tool using JMH command line in different ways:
+ *
+ * To use the settings shown in the main() function, use:
+ * $ java -cp target/benchmarks.jar org.apache.hive.benchmark.calcite.FieldTrimmerBench
+ *
+ * To use the default settings used by JMH, use:
+ * $ java -jar target/benchmarks.jar org.apache.hive.benchmark.calcite.FieldTrimmerBench
+ */
+@State(Scope.Thread)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Fork(1)
+public class FieldTrimmerBench {
+
+ RelOptCluster relOptCluster;
+ RelBuilder relBuilder;
+ RelNode root;
+ org.apache.calcite.sql2rel.RelFieldTrimmer cft;
+ HiveRelFieldTrimmer ft;
+ HiveRelFieldTrimmer hft;
+
+ @Setup(Level.Trial)
+ public void initTrial() {
+ // Init cluster and builder
+ final RelOptPlanner planner = CalcitePlanner.createPlanner(new HiveConf());
+ final RexBuilder rexBuilder = new RexBuilder(
+ new JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
+ relOptCluster = RelOptCluster.create(planner, rexBuilder);
+ relBuilder = HiveRelFactories.HIVE_BUILDER.create(relOptCluster, null);
+ // Create operator tree
+ DummyNode0 d0 = new DummyNode0(relOptCluster, relOptCluster.traitSet());
+ DummyNode1 d1 = new DummyNode1(relOptCluster, relOptCluster.traitSet());
+ DummyNode2 d2 = new DummyNode2(relOptCluster, relOptCluster.traitSet());
+ DummyNode3 d3 = new DummyNode3(relOptCluster, relOptCluster.traitSet());
+ DummyNode4 d4 = new DummyNode4(relOptCluster, relOptCluster.traitSet(), d0);
+ DummyNode5 d5 = new DummyNode5(relOptCluster, relOptCluster.traitSet(), d1);
+ DummyNode6 d6 = new DummyNode6(relOptCluster, relOptCluster.traitSet(), d2);
+ DummyNode7 d7 = new DummyNode7(relOptCluster, relOptCluster.traitSet(), d3);
+ DummyNode8 d8 = new DummyNode8(relOptCluster, relOptCluster.traitSet(), d4, d5);
+ DummyNode9 d9 = new DummyNode9(relOptCluster, relOptCluster.traitSet(), d6, d7);
+ root = new DummyNode9(relOptCluster, relOptCluster.traitSet(), d8, d9);
+ }
+
+ @Benchmark
+ @BenchmarkMode({Mode.Throughput, Mode.AverageTime})
+ @Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS)
+ public void baseRelFieldTrimmer() {
+ // We initialize the field trimmer for every execution of the benchmark
+ cft = new org.apache.calcite.sql2rel.RelFieldTrimmer(null, relBuilder);
+ cft.trim(root);
+ cft = null;
+ }
+
+ @Benchmark
+ @BenchmarkMode({Mode.Throughput, Mode.AverageTime})
+ @Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS)
+ public void modBaseRelFieldTrimmer() {
+ // We initialize the field trimmer for every execution of the benchmark
+ ft = HiveRelFieldTrimmer.get(false, false);
+ ft.trim(relBuilder, root);
+ ft = null;
+ }
+
+ @Benchmark
+ @BenchmarkMode({Mode.Throughput, Mode.AverageTime})
+ @Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS)
+ public void hiveRelFieldTrimmer() {
+ // We initialize the field trimmer for every execution of the benchmark
+ hft = HiveRelFieldTrimmer.get(false);
+ hft.trim(relBuilder, root);
+ hft = null;
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder().include(".*" + FieldTrimmerBench.class.getSimpleName() +
+ ".*").build();
+ new Runner(opt).run();
+ }
+
+ // ~ 10 rel node classes to use in the benchmark.
+
+ private class DummyNode0 extends AbstractRelNode {
+ protected DummyNode0(RelOptCluster cluster, RelTraitSet traits) {
+ super(cluster, cluster.traitSet());
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode1 extends AbstractRelNode {
+ protected DummyNode1(RelOptCluster cluster, RelTraitSet traits) {
+ super(cluster, cluster.traitSet());
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode2 extends AbstractRelNode {
+ protected DummyNode2(RelOptCluster cluster, RelTraitSet traits) {
+ super(cluster, cluster.traitSet());
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode3 extends AbstractRelNode {
+ protected DummyNode3(RelOptCluster cluster, RelTraitSet traits) {
+ super(cluster, cluster.traitSet());
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode4 extends SingleRel {
+ protected DummyNode4(RelOptCluster cluster, RelTraitSet traits, RelNode input) {
+ super(cluster, cluster.traitSet(), input);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode5 extends SingleRel {
+ protected DummyNode5(RelOptCluster cluster, RelTraitSet traits, RelNode input) {
+ super(cluster, cluster.traitSet(), input);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode6 extends SingleRel {
+ protected DummyNode6(RelOptCluster cluster, RelTraitSet traits, RelNode input) {
+ super(cluster, cluster.traitSet(), input);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode7 extends SingleRel {
+ protected DummyNode7(RelOptCluster cluster, RelTraitSet traits, RelNode input) {
+ super(cluster, cluster.traitSet(), input);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode8 extends BiRel {
+ protected DummyNode8(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right) {
+ super(cluster, cluster.traitSet(), left, right);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+
+ private class DummyNode9 extends BiRel {
+ protected DummyNode9(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right) {
+ super(cluster, cluster.traitSet(), left, right);
+ }
+
+ protected RelDataType deriveRowType() {
+ return new RelRecordType(Lists.newArrayList());
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
index 7d55f640ec..7ad3214c66 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
@@ -94,53 +94,14 @@
HiveRelMdPredicates.SOURCE,
JaninoRelMetadataProvider.DEFAULT)));
- /**
- * This is the list of operators that are specifically used in Hive and
- * should be loaded by the metadata providers.
- */
- private static final List> HIVE_REL_NODE_CLASSES =
- ImmutableList.of(
- RelNode.class,
- AbstractRelNode.class,
- RelSubset.class,
- HepRelVertex.class,
- ConverterImpl.class,
- AbstractConverter.class,
-
- HiveTableScan.class,
- HiveAggregate.class,
- HiveExcept.class,
- HiveFilter.class,
- HiveIntersect.class,
- HiveJoin.class,
- HiveMultiJoin.class,
- HiveProject.class,
- HiveRelNode.class,
- HiveSemiJoin.class,
- HiveSortExchange.class,
- HiveSortLimit.class,
- HiveTableFunctionScan.class,
- HiveUnion.class,
-
- DruidQuery.class,
-
- HiveJdbcConverter.class,
- JdbcHiveTableScan.class,
- JdbcAggregate.class,
- JdbcFilter.class,
- JdbcJoin.class,
- JdbcProject.class,
- JdbcSort.class,
- JdbcUnion.class);
-
private final RelMetadataProvider metadataProvider;
- public HiveDefaultRelMetadataProvider(HiveConf hiveConf) {
- this.metadataProvider = init(hiveConf);
+ public HiveDefaultRelMetadataProvider(HiveConf hiveConf, List> nodeClasses) {
+ this.metadataProvider = init(hiveConf, nodeClasses);
}
- private RelMetadataProvider init(HiveConf hiveConf) {
+ private RelMetadataProvider init(HiveConf hiveConf, List> nodeClasses) {
// Create cost metadata provider
if (HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
&& HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_EXTENDED_COST_MODEL)) {
@@ -167,7 +128,10 @@ private RelMetadataProvider init(HiveConf hiveConf) {
HiveRelMdPredicates.SOURCE,
JaninoRelMetadataProvider.DEFAULT)));
- metadataProvider.register(HIVE_REL_NODE_CLASSES);
+ if (nodeClasses != null) {
+ // If classes were passed, pre-register them
+ metadataProvider.register(nodeClasses);
+ }
return metadataProvider;
}
@@ -184,11 +148,10 @@ public RelMetadataProvider getMetadataProvider() {
* additional Hive classes (compared to Calcite core classes) that may
* be visited during the planning phase.
*/
- public static void initializeMetadataProviderClass() {
+ public static void initializeMetadataProviderClass(List> nodeClasses) {
// This will register the classes in the default Janino implementation
- JaninoRelMetadataProvider.DEFAULT.register(
- HiveDefaultRelMetadataProvider.HIVE_REL_NODE_CLASSES);
+ JaninoRelMetadataProvider.DEFAULT.register(nodeClasses);
// This will register the classes in the default Hive implementation
- DEFAULT.register(HiveDefaultRelMetadataProvider.HIVE_REL_NODE_CLASSES);
+ DEFAULT.register(nodeClasses);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
index f71d3f0e33..04b3888a25 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
@@ -79,7 +79,8 @@
public static final RelBuilderFactory HIVE_BUILDER =
HiveRelBuilder.proto(
- Contexts.of(HIVE_PROJECT_FACTORY,
+ Contexts.of(
+ HIVE_PROJECT_FACTORY,
HIVE_FILTER_FACTORY,
HIVE_JOIN_FACTORY,
HIVE_SEMI_JOIN_FACTORY,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java
index ac050df30b..73ff1bccf2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java
@@ -63,9 +63,8 @@ public void onMatch(RelOptRuleCall call) {
final HepPlanner tmpPlanner = new HepPlanner(PROGRAM);
tmpPlanner.setRoot(node);
node = tmpPlanner.findBestExp();
- final HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
- relBuilderFactory.create(node.getCluster(), null), fetchStats);
- call.transformTo(fieldTrimmer.trim(node));
+ call.transformTo(
+ HiveRelFieldTrimmer.get(fetchStats).trim(call.builder(), node));
triggered = true;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReflectUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReflectUtil.java
new file mode 100644
index 0000000000..5e327da023
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReflectUtil.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import com.google.common.collect.ImmutableList;
+import java.lang.invoke.CallSite;
+import java.lang.invoke.LambdaMetafactory;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.calcite.util.ReflectUtil;
+import org.apache.calcite.util.ReflectUtil.MethodDispatcher;
+import org.apache.calcite.util.ReflectiveVisitor;
+
+/**
+ * Static utilities for Java reflection. This is based on Calcite
+ * {@link ReflectUtil}. It contains methods to wrap a Calcite dispatcher
+ * (based on reflection) into a Hive dispatcher as well as a Hive
+ * dispatcher implementation based on {@link LambdaMetafactory}.
+ */
+public class HiveReflectUtil {
+
+ /**
+ * Creates a Hive dispatcher that wraps a Calcite one.
+ */
+ protected static MethodDispatcherWrapper createCalciteMethodDispatcherWrapper(
+ final MethodDispatcher methodDispatcher) {
+ return new MethodDispatcherWrapper<>(methodDispatcher);
+ }
+
+ /**
+ * Creates a dispatcher for calls to a single multi-method on a particular
+ * object.
+ *
+ *
Calls to that multi-method are resolved by looking for a method on
+ * the runtime type of that object, with the required name, and with
+ * the correct type or a subclass for the first argument, and precisely the
+ * same types for other arguments.
+ *
+ *
For instance, a dispatcher created for the method
+ *
+ *
(because Car and Bus are subclasses of Vehicle, and they occur in the
+ * polymorphic first argument) but not the method
+ *
+ *
String foo(Car, int, ArrayList)
+ *
+ *
(only the first argument is polymorphic).
+ *
+ *
You must create an implementation of the method for the base class.
+ * Otherwise throws {@link IllegalArgumentException}.
+ *
+ * @param returnClazz Return type of method
+ * @param visitor Object on which to invoke the method
+ * @param methodName Name of method
+ * @param arg0Clazz Base type of argument zero
+ * @param otherArgClasses Types of remaining arguments
+ */
+ protected static HiveMethodDispatcher createMethodDispatcher(
+ final Class returnClazz,
+ final ReflectiveVisitor visitor,
+ final String methodName,
+ final Class arg0Clazz,
+ final Class... otherArgClasses) {
+ final List otherArgClassList =
+ ImmutableList.copyOf(otherArgClasses);
+ final VisitDispatcher dispatcher =
+ createDispatcher((Class) visitor.getClass(), arg0Clazz);
+ return new HiveMethodDispatcher<>(dispatcher, returnClazz, visitor, methodName,
+ arg0Clazz, otherArgClassList);
+ }
+
+ /**
+ * Creates a dispatcher for calls to {@link VisitDispatcher#lookupVisitFunc}. The
+ * dispatcher caches methods between invocations and it is thread-safe.
+ *
+ * @param visitorBaseClazz Visitor base class
+ * @param visiteeBaseClazz Visitee base class
+ * @return cache of methods
+ */
+ private static VisitDispatcher createDispatcher(
+ final Class visitorBaseClazz,
+ final Class visiteeBaseClazz) {
+ assert ReflectiveVisitor.class.isAssignableFrom(visitorBaseClazz);
+ assert Object.class.isAssignableFrom(visiteeBaseClazz);
+ return new VisitDispatcher<>();
+ }
+
+ private static Class extends VarArgsFunc> getVarArgsFuncClass(int length) {
+ switch (length) {
+ case 1:
+ return VarArgsFunc1.class;
+ case 2:
+ return VarArgsFunc2.class;
+ case 3:
+ return VarArgsFunc3.class;
+ case 4:
+ return VarArgsFunc4.class;
+ default:
+ throw new RuntimeException("Unsupported function with length " + length);
+ }
+ }
+
+ private static VarArgsFunc getVarArgsFunc(int length, CallSite site) throws Throwable {
+ switch (length) {
+ case 1:
+ return (VarArgsFunc1) site.getTarget().invokeExact();
+ case 2:
+ return (VarArgsFunc2) site.getTarget().invokeExact();
+ case 3:
+ return (VarArgsFunc3) site.getTarget().invokeExact();
+ case 4:
+ return (VarArgsFunc4) site.getTarget().invokeExact();
+ default:
+ throw new RuntimeException("Unsupported function with length " + length);
+ }
+ }
+
+ protected static class VisitDispatcher {
+ final Map, VarArgsFunc> map = new ConcurrentHashMap<>();
+
+ public VarArgsFunc lookupVisitFunc(
+ Class extends R> visitorClass,
+ Class extends E> visiteeClass,
+ String visitMethodName,
+ List additionalParameterTypes)
+ throws Throwable {
+ final List