diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6de6ed6..43de1a9 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2261,7 +2261,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Choices between memory, ssd and default"), HIVE_QUERY_LIFETIME_HOOKS("hive.query.lifetime.hooks", "", "A comma separated list of hooks which implement QueryLifeTimeHook. These will be triggered" + - " before/after query compilation and before/after query execution, in the order specified"), + " before/after query compilation and before/after query execution, in the order specified." + + "Implementations of QueryLifeTimeHookWithParseHooks can also be specified in this list. If they are" + + "specified then they will be invoked during pre and post query parsing"), HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", "", "A comma separated list of hooks which implement HiveDriverRunHook. Will be run at the beginning " + "and end of Driver.run, these will be run in the order specified."), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 630df43..cc1f7d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -66,12 +66,14 @@ import org.apache.hadoop.hive.ql.hooks.Hook; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookUtils; +import org.apache.hadoop.hive.ql.hooks.HooksLoader; import org.apache.hadoop.hive.ql.hooks.MetricsQueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContext; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContextImpl; +import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookWithParseHooks; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; @@ -96,6 +98,7 @@ import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; @@ -178,7 +181,8 @@ private QueryState queryState; // Query hooks that execute before compilation and after execution - private List queryHooks; + private QueryLifeTimeHookRunner queryLifeTimeHookRunner; + private final HooksLoader hooksLoader; public enum DriverState { INITIALIZED, @@ -353,6 +357,8 @@ public Driver(QueryState queryState, String userName) { isParallelEnabled = (conf != null) && HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_PARALLEL_COMPILATION); this.userName = userName; + this.queryLifeTimeHookRunner = new QueryLifeTimeHookRunner(conf); + this.hooksLoader = new HooksLoader(conf); } /** @@ -432,6 +438,7 @@ public int compile(String command, boolean resetTaskIds, boolean deferClose) { // Whether any error occurred during query compilation. Used for query lifetime hook. boolean compileError = false; + try { // Initialize the transaction manager. This must be done before analyze is called. @@ -464,27 +471,29 @@ public void run() { ctx.setCmd(command); ctx.setHDFSCleanup(true); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARSE); - ASTNode tree = ParseUtils.parse(command, ctx); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); - // Trigger query hook before compilation - queryHooks = loadQueryHooks(); - if (queryHooks != null && !queryHooks.isEmpty()) { - QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); - qhc.setHiveConf(conf); - qhc.setCommand(command); + queryLifeTimeHookRunner.loadQueryLifeTimeHooks(hooksLoader, console); + queryLifeTimeHookRunner.runBeforeParseHook(command); - for (QueryLifeTimeHook hook : queryHooks) { - hook.beforeCompile(qhc); - } + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARSE); + boolean parseError = false; + ASTNode tree; + try { + tree = ParseUtils.parse(command, ctx); + } catch (ParseException e) { + parseError = true; + throw e; + } finally { + queryLifeTimeHookRunner.runAfterParseHook(command, parseError); } + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); + + queryLifeTimeHookRunner.runBeforeCompileHook(command); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ANALYZE); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, tree); List saHooks = - getHooks(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, - HiveSemanticAnalyzerHook.class); + hooksLoader.getHooks(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, console); // Flush the metastore cache. This assures that we don't pick up objects from a previous // query running in this same thread. This has to be done after we get our semantic @@ -604,14 +613,7 @@ public void run() { // Trigger post compilation hook. Note that if the compilation fails here then // before/after execution hook will never be executed. try { - if (queryHooks != null && !queryHooks.isEmpty()) { - QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); - qhc.setHiveConf(conf); - qhc.setCommand(command); - for (QueryLifeTimeHook hook : queryHooks) { - hook.afterCompile(qhc, compileError); - } - } + queryLifeTimeHookRunner.runAfterCompilationHook(command, compileError); } catch (Exception e) { LOG.warn("Failed when invoking query after-compilation hook.", e); } @@ -664,19 +666,6 @@ private boolean isInterrupted() { } } - private List loadQueryHooks() throws Exception { - List hooks = new ArrayList<>(); - - if (conf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { - hooks.add(new MetricsQueryLifeTimeHook()); - } - List propertyDefinedHoooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); - if (propertyDefinedHoooks != null) { - Iterables.addAll(hooks, propertyDefinedHoooks); - } - return hooks; - } - private ImmutableMap dumpMetaCallTimingWithoutEx(String phase) { try { return Hive.get().dumpAndClearMetaCallTiming(phase); @@ -1436,8 +1425,7 @@ private CommandProcessorResponse runInternal(String command, boolean alreadyComp // Get all the driver run hooks and pre-execute them. List driverRunHooks; try { - driverRunHooks = getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS, - HiveDriverRunHook.class); + driverRunHooks = hooksLoader.getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS, console); for (HiveDriverRunHook driverRunHook : driverRunHooks) { driverRunHook.preDriverRun(hookContext); } @@ -1662,34 +1650,6 @@ private CommandProcessorResponse createProcessorResponse(int ret) { return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError); } - /** - * Returns a set of hooks specified in a configuration variable. - * See getHooks(HiveConf.ConfVars hookConfVar, Class clazz) - */ - private List getHooks(HiveConf.ConfVars hookConfVar) throws Exception { - return getHooks(hookConfVar, Hook.class); - } - - /** - * Returns the hooks specified in a configuration variable. - * - * @param hookConfVar The configuration variable specifying a comma separated list of the hook - * class names. - * @param clazz The super type of the hooks. - * @return A list of the hooks cast as the type specified in clazz, in the order - * they are listed in the value of hookConfVar - * @throws Exception - */ - private List getHooks(ConfVars hookConfVar, - Class clazz) throws Exception { - try { - return HookUtils.getHooks(conf, hookConfVar, clazz); - } catch (ClassNotFoundException e) { - console.printError(hookConfVar.varname + " Class not found:" + e.getMessage()); - throw e; - } - } - public int execute() throws CommandNeedRetryException { return execute(false); } @@ -1754,7 +1714,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger); hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK); - for (Hook peh : getHooks(HiveConf.ConfVars.PREEXECHOOKS)) { + for (Hook peh : hooksLoader.getHooks(HiveConf.ConfVars.PREEXECHOOKS, console)) { if (peh instanceof ExecuteWithHookContext) { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); @@ -1772,16 +1732,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { } // Trigger query hooks before query execution. - if (queryHooks != null && !queryHooks.isEmpty()) { - QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); - qhc.setHiveConf(conf); - qhc.setCommand(queryStr); - qhc.setHookContext(hookContext); - - for (QueryLifeTimeHook hook : queryHooks) { - hook.beforeExecution(qhc); - } - } + queryLifeTimeHookRunner.runBeforeExecutionHook(queryStr, hookContext); setQueryDisplays(plan.getRootTasks()); int mrJobs = Utilities.getMRTasks(plan.getRootTasks()).size(); @@ -1939,7 +1890,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { hookContext.setHookType(HookContext.HookType.POST_EXEC_HOOK); // Get all the post execution hooks and execute them. - for (Hook peh : getHooks(HiveConf.ConfVars.POSTEXECHOOKS)) { + for (Hook peh : hooksLoader.getHooks(HiveConf.ConfVars.POSTEXECHOOKS, console)) { if (peh instanceof ExecuteWithHookContext) { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); @@ -1994,16 +1945,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { } finally { // Trigger query hooks after query completes its execution. try { - if (queryHooks != null && !queryHooks.isEmpty()) { - QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); - qhc.setHiveConf(conf); - qhc.setCommand(queryStr); - qhc.setHookContext(hookContext); - - for (QueryLifeTimeHook hook : queryHooks) { - hook.afterExecution(qhc, executionError); - } - } + queryLifeTimeHookRunner.runAfterExecutionHook(queryStr, hookContext, executionError); } catch (Exception e) { LOG.warn("Failed when invoking query after execution hook", e); } @@ -2125,7 +2067,7 @@ private void invokeFailureHooks(PerfLogger perfLogger, hookContext.setErrorMessage(errorMessage); hookContext.setException(exception); // Get all the failure execution hooks and execute them. - for (Hook ofh : getHooks(HiveConf.ConfVars.ONFAILUREHOOKS)) { + for (Hook ofh : hooksLoader.getHooks(HiveConf.ConfVars.ONFAILUREHOOKS, console)) { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.FAILURE_HOOK + ofh.getClass().getName()); ((ExecuteWithHookContext) ofh).run(hookContext); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java new file mode 100644 index 0000000..c3fb5f4 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryLifeTimeHookRunner.java @@ -0,0 +1,127 @@ +package org.apache.hadoop.hive.ql; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.Iterables; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.hooks.HookContext; +import org.apache.hadoop.hive.ql.hooks.HooksLoader; +import org.apache.hadoop.hive.ql.hooks.MetricsQueryLifeTimeHook; +import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook; +import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContext; +import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContextImpl; +import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookWithParseHooks; +import org.apache.hadoop.hive.ql.session.SessionState; + + +/** + * A runner class for {@link QueryLifeTimeHook}s and {@link QueryLifeTimeHookWithParseHooks}. + */ +class QueryLifeTimeHookRunner { + + private final HiveConf conf; + private List queryHooks; + + QueryLifeTimeHookRunner(HiveConf conf) { + this.conf = conf; + } + + void loadQueryLifeTimeHooks(HooksLoader hooksLoader, + SessionState.LogHelper console) throws IllegalAccessException, ClassNotFoundException, InstantiationException { + if (queryHooks == null) { + queryHooks = new ArrayList<>(); + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { + queryHooks.add(new MetricsQueryLifeTimeHook()); + } + List propertyDefinedHoooks = hooksLoader.getHooks( + HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS, console); + if (propertyDefinedHoooks != null) { + Iterables.addAll(queryHooks, propertyDefinedHoooks); + } + } + } + + void runBeforeParseHook(String command) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + + for (QueryLifeTimeHook hook : queryHooks) { + if (hook instanceof QueryLifeTimeHookWithParseHooks) { + ((QueryLifeTimeHookWithParseHooks) hook).beforeParse(qhc); + } + } + } + } + + void runAfterParseHook(String command, boolean parseError) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + + for (QueryLifeTimeHook hook : queryHooks) { + if (hook instanceof QueryLifeTimeHookWithParseHooks) { + ((QueryLifeTimeHookWithParseHooks) hook).afterParse(qhc, parseError); + } + } + } + } + + void runBeforeCompileHook(String command) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + + for (QueryLifeTimeHook hook : queryHooks) { + hook.beforeCompile(qhc); + } + } + } + + void runAfterCompilationHook(String command, boolean compileError) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + + for (QueryLifeTimeHook hook : queryHooks) { + hook.afterCompile(qhc, compileError); + } + } + } + + void runBeforeExecutionHook(String command, HookContext hookContext) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + qhc.setHookContext(hookContext); + + for (QueryLifeTimeHook hook : queryHooks) { + hook.beforeExecution(qhc); + } + } + } + + void runAfterExecutionHook(String queryStr, HookContext hookContext, boolean executionError) { + if (containsHooks()) { + QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(queryStr); + qhc.setHookContext(hookContext); + + for (QueryLifeTimeHook hook : queryHooks) { + hook.afterExecution(qhc, executionError); + } + } + } + + private boolean containsHooks() { + return queryHooks != null && !queryHooks.isEmpty(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java index 2f0bd88..4380fe3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,66 +18,26 @@ package org.apache.hadoop.hive.ql.hooks; -import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.exec.Utilities; -public class HookUtils { - /** - * Returns the hooks specified in a configuration variable. The hooks are returned - * in a list in the order they were specified in the configuration variable. - * - * @param conf Configuration object - * @param hookConfVar The configuration variable specifying a comma separated list - * of the hook class names. - * @param clazz The super type of the hooks. - * @return A list of the hooks cast as the type specified in clazz, - * in the order they are listed in the value of hookConfVar - * @throws ClassNotFoundException - * @throws IllegalAccessException - * @throws InstantiationException - */ - public static List getHooks(HiveConf conf, - ConfVars hookConfVar, Class clazz) - throws InstantiationException, IllegalAccessException, ClassNotFoundException { - String csHooks = conf.getVar(hookConfVar); - List hooks = new ArrayList(); - if (csHooks == null) { - return hooks; - } - csHooks = csHooks.trim(); - if (csHooks.equals("")) { - return hooks; - } - - String[] hookClasses = csHooks.split(","); - for (String hookClass : hookClasses) { - T hook = (T) Class.forName(hookClass.trim(), true, - Utilities.getSessionSpecifiedClassLoader()).newInstance(); - hooks.add(hook); - } - - return hooks; - } +public class HookUtils { public static String redactLogString(HiveConf conf, String logString) - throws InstantiationException, IllegalAccessException, ClassNotFoundException { + throws InstantiationException, IllegalAccessException, ClassNotFoundException { String redactedString = logString; if (conf != null && logString != null) { - List queryRedactors = getHooks(conf, ConfVars.QUERYREDACTORHOOKS, Redactor.class); + List queryRedactors = new HooksLoader(conf).getHooks(ConfVars.QUERYREDACTORHOOKS); for (Redactor redactor : queryRedactors) { redactor.setConf(conf); redactedString = redactor.redactQuery(redactedString); } } - return redactedString; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HooksLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HooksLoader.java new file mode 100644 index 0000000..fdaaa15 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HooksLoader.java @@ -0,0 +1,75 @@ +package org.apache.hadoop.hive.ql.hooks; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.session.SessionState; + + +/** + * A loader class for {@link Hook}s. + */ +public class HooksLoader { + + private final HiveConf conf; + + public HooksLoader(HiveConf conf) { + this.conf = conf; + } + + /** + * Returns the hooks specified in a configuration variable. + * + * @param hookConfVar The configuration variable specifying a comma separated list of the hook + * class names. + * @return A list of the hooks cast as the type specified in clazz, in the order + * they are listed in the value of hookConfVar + * @throws Exception + */ + public List getHooks(HiveConf.ConfVars hookConfVar, + SessionState.LogHelper console) throws IllegalAccessException, InstantiationException, ClassNotFoundException { + try { + return getHooks(hookConfVar); + } catch (ClassNotFoundException e) { + console.printError(hookConfVar.varname + " Class not found:" + e.getMessage()); + throw e; + } + } + + /** + * Returns the hooks specified in a configuration variable. The hooks are returned + * in a list in the order they were specified in the configuration variable. + * + * @param hookConfVar The configuration variable specifying a comma separated list + * of the hook class names. + * @return A list of the hooks cast as the type specified in clazz, + * in the order they are listed in the value of hookConfVar + * @throws ClassNotFoundException + * @throws IllegalAccessException + * @throws InstantiationException + */ + public List getHooks(HiveConf.ConfVars hookConfVar) + throws InstantiationException, IllegalAccessException, ClassNotFoundException { + String csHooks = conf.getVar(hookConfVar); + List hooks = new ArrayList(); + if (csHooks == null) { + return hooks; + } + + csHooks = csHooks.trim(); + if (csHooks.equals("")) { + return hooks; + } + + String[] hookClasses = csHooks.split(","); + for (String hookClass : hookClasses) { + T hook = (T) Class.forName(hookClass.trim(), true, + Utilities.getSessionSpecifiedClassLoader()).newInstance(); + hooks.add(hook); + } + + return hooks; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryLifeTimeHookWithParseHooks.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryLifeTimeHookWithParseHooks.java new file mode 100644 index 0000000..1bde191 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryLifeTimeHookWithParseHooks.java @@ -0,0 +1,24 @@ +package org.apache.hadoop.hive.ql.hooks; + + +/** + * Extension of {@link QueryLifeTimeHook} that has hooks for pre and post parsing of a query. + */ +public interface QueryLifeTimeHookWithParseHooks extends QueryLifeTimeHook { + + /** + * Invoked before a query enters the parse phase. + * + * @param ctx the context for the hook + */ + void beforeParse(QueryLifeTimeHookContext ctx); + + /** + * Invoked after a query parsing. Note: if 'hasError' is true, + * the query won't enter the following compilation phase. + * + * @param ctx the context for the hook + * @param hasError whether any error occurred during compilation. + */ + void afterParse(QueryLifeTimeHookContext ctx, boolean hasError); +}