diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 1ef636c..f6eb64b 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -39,6 +39,7 @@ public static final String COMPILE = "compile"; public static final String PARSE = "parse"; public static final String ANALYZE = "semanticAnalyze"; + public static final String OPTIMIZER = "optimizer"; public static final String DO_AUTHORIZATION = "doAuthorization"; public static final String DRIVER_EXECUTE = "Driver.execute"; public static final String INPUT_SUMMARY = "getInputSummary"; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java index 750427a..a649fdf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java @@ -41,7 +41,7 @@ /** * this transformation does bucket map join optimization. */ -public class BucketMapJoinOptimizer implements Transform { +public class BucketMapJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(GroupByOptimizer.class .getName()); @@ -49,7 +49,6 @@ public BucketMapJoinOptimizer() { } - @Override public ParseContext transform(ParseContext pctx) throws SemanticException { Map opRules = new LinkedHashMap(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index f59ab4b..312a02c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -72,7 +72,7 @@ * where T1, T2 and T3 are bucketized/sorted on the same key 'key', we don't need a reducer * to enforce bucketing and sorting */ -public class BucketingSortingReduceSinkOptimizer implements Transform { +public class BucketingSortingReduceSinkOptimizer extends Transform { public BucketingSortingReduceSinkOptimizer() { } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java index 561b8fc..c353e3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java @@ -55,7 +55,7 @@ * this changes the row resolver, the tree is built again. This can be optimized * later to patch the tree. */ -public class ColumnPruner implements Transform { +public class ColumnPruner extends Transform { protected ParseContext pGraphContext; /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java index 136b5e1..6ff151d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java @@ -59,7 +59,7 @@ * 3. Propagate expression: if the expression is an assignment like column=constant, the expression * will be propagate to parents to see if further folding operation is possible. */ -public class ConstantPropagate implements Transform { +public class ConstantPropagate extends Transform { private static final Logger LOG = LoggerFactory.getLogger(ConstantPropagate.class); protected ParseContext pGraphContext; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index c036db0..c63318e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -60,7 +60,7 @@ * Fixed bucket pruning optimizer goes through all the table scans and annotates them * with a bucketing inclusion bit-set. */ -public class FixedBucketPruningOptimizer implements Transform { +public class FixedBucketPruningOptimizer extends Transform { private static final Log LOG = LogFactory .getLog(FixedBucketPruningOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java index 6b04d92..74d6292 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java @@ -61,7 +61,7 @@ * needed by this query. * This optimizer step populates the GlobalLimitCtx which is used later on to prune the inputs. */ -public class GlobalLimitOptimizer implements Transform { +public class GlobalLimitOptimizer extends Transform { private final Logger LOG = LoggerFactory.getLogger(GlobalLimitOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index fe459f7..8f3ce60 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -69,7 +69,7 @@ * of the bucketing and sorting keys of the underlying table in the same order, the * group by can be be performed on the map-side completely. */ -public class GroupByOptimizer implements Transform { +public class GroupByOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(GroupByOptimizer.class .getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java index 114c683..48021cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java @@ -68,7 +68,7 @@ * Note absence of select operator after filter and after join operator. * Also, see : identity_proj_remove.q */ -public class IdentityProjectRemover implements Transform { +public class IdentityProjectRemover extends Transform { private static final Logger LOG = LoggerFactory.getLogger(IdentityProjectRemover.class); @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java index e31dc9b..b385c90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/JoinReorder.java @@ -37,7 +37,7 @@ * largest tag so that they are processed last. In future, once statistics are * implemented, this transformation can also be done based on costs. */ -public class JoinReorder implements Transform { +public class JoinReorder extends Transform { private final Map, Integer> cache = new IdentityHashMap, Integer>(); /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java index e850550..5a55061 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/LimitPushdownOptimizer.java @@ -85,7 +85,7 @@ * REDUCER : 10(b,d)-20(l)-30(c,k,o)-40(j,m)-50(n) * LIMIT 3 : 10(b,d)-20(l) */ -public class LimitPushdownOptimizer implements Transform { +public class LimitPushdownOptimizer extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index b4276e4..2e3f930 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -86,7 +86,7 @@ * the reduce sink operator above the join are converted to map sink operators. In future, once * statistics are implemented, this transformation can also be done based on costs. */ -public class MapJoinProcessor implements Transform { +public class MapJoinProcessor extends Transform { // mapjoin table descriptor contains a key descriptor which needs the field schema // (column type + column name). The column name is not really used anywhere, but it diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java index 2764ae1..37dbe32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java @@ -54,7 +54,7 @@ /** * merges SEL-SEL or FIL-FIL into single operator */ -public class NonBlockingOpDeDupProc implements Transform { +public class NonBlockingOpDeDupProc extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 4a7fc0d..8f48e7d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -224,7 +224,9 @@ public void initialize(HiveConf hiveConf) { */ public ParseContext optimize() throws SemanticException { for (Transform t : transformations) { - pctx = t.transform(pctx); + t.beginPerfLogging(); + pctx = t.transform(pctx); + t.endPerfLogging(t.toString()); } return pctx; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java index f71f37c..aea28e2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PartitionColumnsSeparator.java @@ -62,7 +62,7 @@ * This predicate is in turn used by the partition pruner to prune the columns that are not * part of the original IN(STRUCT(..)..) predicate. */ -public class PartitionColumnsSeparator implements Transform { +public class PartitionColumnsSeparator extends Transform { private static final Log LOG = LogFactory.getLog(PointLookupOptimizer.class); private static final String IN_UDF = diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java index a1a49cd..44972bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PointLookupOptimizer.java @@ -64,7 +64,7 @@ * AND operator children, the optimization might generate an IN clause that uses * structs. */ -public class PointLookupOptimizer implements Transform { +public class PointLookupOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(PointLookupOptimizer.class); private static final String IN_UDF = diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java index 2c473b0..dd679db 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java @@ -54,7 +54,7 @@ * The transformation step that does sample pruning. * */ -public class SamplePruner implements Transform { +public class SamplePruner extends Transform { /** * SamplePrunerCtx. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java index 39e11a2..f71fd3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; // execute final aggregation stage for simple fetch query on fetch task -public class SimpleFetchAggregation implements Transform { +public class SimpleFetchAggregation extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java index 332e53b..bc36122 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java @@ -87,7 +87,7 @@ * Tries to convert simple fetch query to single fetch task, which fetches rows directly * from location of table/partition. */ -public class SimpleFetchOptimizer implements Transform { +public class SimpleFetchOptimizer extends Transform { private final Logger LOG = LoggerFactory.getLogger(SimpleFetchOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java index 64dc48c..9525271 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java @@ -75,7 +75,7 @@ * SkewJoinOptimizer. * */ -public class SkewJoinOptimizer implements Transform { +public class SkewJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(SkewJoinOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index c3553a5..d4be5cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -76,7 +76,7 @@ * into the destination table. This enables reducers to keep only one record writer all the time * thereby reducing the the memory pressure on the reducers. */ -public class SortedDynPartitionOptimizer implements Transform { +public class SortedDynPartitionOptimizer extends Transform { private static final String BUCKET_NUMBER_COL_NAME = "_bucket_number"; @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java index 5aeeeb8..6aa5335 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeBucketMapJoinOptimizer.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc; //try to replace a bucket map join with a sorted merge map join -public class SortedMergeBucketMapJoinOptimizer implements Transform { +public class SortedMergeBucketMapJoinOptimizer extends Transform { private static final Logger LOG = LoggerFactory .getLogger(SortedMergeBucketMapJoinOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index ffe706e..03c1c3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -89,7 +89,7 @@ * Optimizer looks at query plan to determine if it can answer query using statistics * and than change the plan to answer query entirely using statistics stored in metastore. */ -public class StatsOptimizer implements Transform { +public class StatsOptimizer extends Transform { // TODO: [HIVE-6289] while getting stats from metastore, we currently only get one col at // a time; this could be improved - get all necessary columns in advance, then use local. // TODO: [HIVE-6292] aggregations could be done directly in metastore. Hive over MySQL! diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java index 9f957da..9ce9478 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.ql.optimizer; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; /** * Optimizer interface. All the rule-based optimizations implement this @@ -27,7 +29,7 @@ * current parse context (which contains the operator tree among other things), * perform all the optimizations, and then return the updated parse context. */ -public interface Transform { +public abstract class Transform { /** * All transformation steps implement this interface. * @@ -36,5 +38,19 @@ * @return ParseContext * @throws SemanticException */ - ParseContext transform(ParseContext pctx) throws SemanticException; + public abstract ParseContext transform(ParseContext pctx) throws SemanticException; + + public void beginPerfLogging() { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + } + + public void endPerfLogging() { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER); + } + public void endPerfLogging(String additionalInfo) { + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, additionalInfo); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java index 1d0a254..fc6b99d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java @@ -46,7 +46,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -public class HiveOpConverterPostProc implements Transform { +public class HiveOpConverterPostProc extends Transform { private static final Logger LOG = LoggerFactory.getLogger(HiveOpConverterPostProc.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java index 1b5d921..e1927e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java @@ -82,7 +82,7 @@ * Test queries associated with this optimizer are correlationoptimizer1.q to * correlationoptimizer14.q */ -public class CorrelationOptimizer implements Transform { +public class CorrelationOptimizer extends Transform { private static final Logger LOG = LoggerFactory.getLogger(CorrelationOptimizer.class.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 583e72f..8da9ad5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -62,7 +62,7 @@ * * This optimizer removes/replaces child-RS (not parent) which is safer way for DefaultGraphWalker. */ -public class ReduceSinkDeDuplication implements Transform { +public class ReduceSinkDeDuplication extends Transform { private static final String RS = ReduceSinkOperator.getOperatorName(); private static final String GBY = GroupByOperator.getOperatorName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java index ea1ece6..e59127f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java @@ -89,7 +89,7 @@ * For test cases, @see ql_rewrite_gbtoidx.q */ -public class RewriteGBUsingIndex implements Transform { +public class RewriteGBUsingIndex extends Transform { private ParseContext parseContext; private Hive hiveDb; private HiveConf hiveConf; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java index 82e26d9..0c2ff32 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java @@ -53,7 +53,7 @@ * and tables from the plan before it goes through other * optimization phases. */ -public class Generator implements Transform { +public class Generator extends Transform { /* (non-Javadoc) * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.ParseContext) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java index f399a20..1965120 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java @@ -41,7 +41,7 @@ * The transformation step that does list bucketing pruning. * */ -public class ListBucketingPruner implements Transform { +public class ListBucketingPruner extends Transform { static final Logger LOG = LoggerFactory.getLogger(ListBucketingPruner.class.getName()); /* diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java index 0398115..b32d730 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java @@ -49,7 +49,7 @@ * This class annotates each operator with its traits. The OpTraits class * specifies the traits that are populated for each operator. */ -public class AnnotateWithOpTraits implements Transform { +public class AnnotateWithOpTraits extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java index 812ca51..7baede9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PartitionConditionRemover.java @@ -44,7 +44,7 @@ * The transformation step that does partition condition remover. * */ -public class PartitionConditionRemover implements Transform { +public class PartitionConditionRemover extends Transform { // The log private static final Logger LOG = LoggerFactory diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index d99362a..90e9b3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -70,7 +70,7 @@ * The transformation step that does partition pruning. * */ -public class PartitionPruner implements Transform { +public class PartitionPruner extends Transform { // The log public static final String CLASS_NAME = PartitionPruner.class.getName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java index c8b3545..f0c1e86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/AnnotateWithStatistics.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; -public class AnnotateWithStatistics implements Transform { +public class AnnotateWithStatistics extends Transform { @Override public ParseContext transform(ParseContext pctx) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java index 547b486..9a53e0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java @@ -50,7 +50,7 @@ * * The parse Context is not changed. */ -public class UnionProcessor implements Transform { +public class UnionProcessor extends Transform { /** * empty constructor. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index c005b1a..b563316 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -181,6 +182,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.serdeConstants; @@ -866,9 +868,12 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu // Create MD provider HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf); + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, mdProvider.getMetadataProvider()); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Prejoin ordering transformations"); // 3. Apply join order optimizations: reordering MST algorithm // If join optimizations failed because of missing stats, we continue with diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index f563b99..0d2f507 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; @@ -86,6 +87,7 @@ import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; /** @@ -112,7 +114,8 @@ public void init(HiveConf conf, LogHelper console, Hive db) { @Override protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, Set outputs) throws SemanticException { - + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); // Create the context for the walker OptimizeTezProcContext procCtx = new OptimizeTezProcContext(conf, pCtx, inputs, outputs); @@ -128,6 +131,7 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // after the stats phase we might have some cyclic dependencies that we need // to take care of. runCycleAnalysisForPartitionPruning(procCtx, inputs, outputs); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Tez compiler"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java index 2312798..23fa2aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java @@ -79,7 +79,7 @@ * etc) change the columns as data flows through them. In such cases the column * references are replaced by the corresponding expression in the input data. */ -public class PredicatePushDown implements Transform { +public class PredicatePushDown extends Transform { private static final Logger LOG = LoggerFactory.getLogger(PredicatePushDown.class); private ParseContext pGraphContext; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java index fb76d5d..b091ebc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java @@ -56,7 +56,7 @@ /** * propagates filters to other aliases based on join condition */ -public class PredicateTransitivePropagate implements Transform { +public class PredicateTransitivePropagate extends Transform { private ParseContext pGraphContext; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java index 7aa91f2..accfb3b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java @@ -60,7 +60,7 @@ /** * creates synthetic predicates that represent "IN (keylist other table)" */ -public class SyntheticJoinPredicate implements Transform { +public class SyntheticJoinPredicate extends Transform { private static transient Logger LOG = LoggerFactory.getLogger(SyntheticJoinPredicate.class.getName());