diff --git ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index eedd652..f202991 100644 --- ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -74,6 +74,8 @@ public static final String SPARK_RUN_STAGE = "SparkRunStage."; public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators"; public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree"; + public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree"; + public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree"; public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable."; protected static final ThreadLocal perfLogger = new ThreadLocal(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 10e1a57..5227d92 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -110,6 +110,7 @@ public void init(HiveConf conf, LogHelper console, Hive db) { @Override protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, Set outputs) throws SemanticException { + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); // Sequence of TableScan operators to be walked Deque> deque = new LinkedList>(); deque.addAll(pCtx.getTopOps().values()); @@ -134,6 +135,7 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, ArrayList topNodes = new ArrayList(); topNodes.addAll(pCtx.getTopOps().values()); ogw.startWalking(topNodes, null); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); } /** @@ -297,6 +299,7 @@ protected void decideExecMode(List> rootTasks, Cont @Override protected void optimizeTaskPlan(List> rootTasks, ParseContext pCtx, Context ctx) throws SemanticException { + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE); PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks, pCtx.getFetchTask()); @@ -339,6 +342,8 @@ protected void optimizeTaskPlan(List> rootTasks, Pa } else { LOG.debug("Skipping stage id rearranger"); } + + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE); return; } }