diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e92466f..1bd19fe 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2108,7 +2108,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.exec.temporary.table.storage", "default", new StringSet("memory", "ssd", "default"), "Define the storage policy for temporary tables." + "Choices between memory, ssd and default"), - + HIVE_QUERY_HOOKS("hive.query.hooks", "", + "A comma separated list of hooks which implement QueryHook. These will be triggered" + + " before query compilation and after query execution, in the order specified"), HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", "", "A comma separated list of hooks which implement HiveDriverRunHook. Will be run at the beginning " + "and end of Driver.run, these will be run in the order specified."), diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 14f221a..42a094c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -63,6 +63,9 @@ import org.apache.hadoop.hive.ql.hooks.HookUtils; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; +import org.apache.hadoop.hive.ql.hooks.QueryHook; +import org.apache.hadoop.hive.ql.hooks.QueryHookContext; +import org.apache.hadoop.hive.ql.hooks.QueryHookContextImpl; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; @@ -403,6 +406,17 @@ public void run() { tree = ParseUtils.findRootNonNullToken(tree); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); + // Trigger query hook before compilation + List queryHooks = getHooks(ConfVars.HIVE_QUERY_HOOKS, QueryHook.class); + if (queryHooks != null && !queryHooks.isEmpty()) { + QueryHookContext qhc = new QueryHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(command); + + for (QueryHook hook : queryHooks) { + hook.pre(qhc); + } + } perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ANALYZE); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, tree); @@ -1714,6 +1728,24 @@ public int execute() throws CommandNeedRetryException { } } + // Trigger query hooks after query completion. + List queryHooks = null; + try { + queryHooks = getHooks(ConfVars.HIVE_QUERY_HOOKS, QueryHook.class); + } catch (Exception e) { + // Pass. The error message is already printed to console in getHooks(). + } + + // TODO: perhaps we should add HookContext to the context? but this overlaps with the above.. + if (queryHooks != null && !queryHooks.isEmpty()) { + QueryHookContext qhc = new QueryHookContextImpl(); + qhc.setHiveConf(conf); + qhc.setCommand(ctx.getCmd()); + + for (QueryHook hook : queryHooks) { + hook.post(qhc); + } + } if (SessionState.get() != null) { SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE, diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHook.java new file mode 100644 index 0000000..0efc2c9 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHook.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.hadoop.hive.ql.hooks; + +/** + * A type of hook which triggers before query compilation and after query execution. + */ +public interface QueryHook extends Hook { + + /** + * Invoked before a query enters the compilation phase. + * + * @param ctx the context for the hook + */ + void pre(QueryHookContext ctx); + + /** + * Invoked after query execution. + * + * @param ctx the context for the hook + */ + void post(QueryHookContext ctx); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContext.java ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContext.java new file mode 100644 index 0000000..5f674cc --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContext.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * Hook context for {@link QueryHook}. + */ +public interface QueryHookContext { + /** + * Get the current Hive configuration + * + * @return the Hive configuration being used + */ + HiveConf getHiveConf(); + + /** + * Set Hive configuration + */ + void setHiveConf(HiveConf conf); + + /** + * Get the current command. + * + * @return the current query command + */ + String getCommand(); + + /** + * Set the current command + * + * @param command the query command to set + */ + void setCommand(String command); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContextImpl.java ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContextImpl.java new file mode 100644 index 0000000..3538a13 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/QueryHookContextImpl.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.conf.HiveConf; + +public class QueryHookContextImpl implements QueryHookContext { + private HiveConf conf; + private String command; + + @Override + public HiveConf getHiveConf() { + return conf; + } + + @Override + public void setHiveConf(HiveConf conf) { + this.conf = conf; + } + + @Override + public String getCommand() { + return command; + } + + @Override + public void setCommand(String command) { + this.command = command; + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHook.java new file mode 100644 index 0000000..1e9d1ae --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHook.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.ql.session.SessionState; + +public class TestQueryHook implements QueryHook { + @Override + public void pre(QueryHookContext ctx) { + SessionState.getConsole().getOutStream().println("pre: " + ctx.getCommand()); + } + + @Override + public void post(QueryHookContext ctx) { + SessionState.getConsole().getOutStream().println("post: " + ctx.getCommand()); + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java new file mode 100644 index 0000000..8f4d2c3 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/TestQueryHooks.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class TestQueryHooks { + private static HiveConf conf; + private static QueryHookContext[] ctxs; + + @BeforeClass + public static void setUpBeforeClass() { + conf = new HiveConf(TestQueryHooks.class); + conf.setVar(HiveConf.ConfVars.HIVE_QUERY_HOOKS, TestHook.class.getName()); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + } + + @Before + public void setUpBefore() { + ctxs = new QueryHookContext[2]; + } + + @Test + public void testQueryHookPre() throws Exception { + Driver driver = createDriver(); + int ret = driver.compile("SELECT 1"); + assertEquals("Expected compilation to succeed", 0, ret); + assertNotNull(ctxs[0]); + assertNull(ctxs[1]); + assertEquals("SELECT 1", ctxs[0].getCommand()); + } + + @Test + public void testQueryHook() throws Exception { + Driver driver = createDriver(); + int ret = driver.run("SELECT 1").getResponseCode(); + assertEquals("Expected query to run", 0, ret); + assertNotNull(ctxs[0]); + assertNotNull(ctxs[1]); + assertEquals("SELECT 1", ctxs[0].getCommand()); + assertEquals("SELECT 1", ctxs[1].getCommand()); + } + + private static Driver createDriver() { + SessionState.start(conf); + Driver driver = new Driver(conf); + driver.init(); + return driver; + } + + /** + * Testing hook which just saves the context + */ + private static class TestHook implements QueryHook { + public TestHook() { + } + + @Override + public void pre(QueryHookContext ctx) { + ctxs[0] = ctx; + } + + @Override + public void post(QueryHookContext ctx) { + ctxs[1] = ctx; + } + } +} diff --git ql/src/test/queries/clientpositive/query_hook.q ql/src/test/queries/clientpositive/query_hook.q new file mode 100644 index 0000000..47b5e96 --- /dev/null +++ ql/src/test/queries/clientpositive/query_hook.q @@ -0,0 +1,3 @@ +SET hive.query.hooks=org.apache.hadoop.hive.ql.hooks.TestQueryHook; + +SELECT * FROM src LIMIT 1; diff --git ql/src/test/results/clientpositive/query_hook.q.out ql/src/test/results/clientpositive/query_hook.q.out new file mode 100644 index 0000000..40013c8 --- /dev/null +++ ql/src/test/results/clientpositive/query_hook.q.out @@ -0,0 +1,15 @@ +pre: + +SELECT * FROM src LIMIT 1 +PREHOOK: query: SELECT * FROM src LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM src LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +post: + +SELECT * FROM src LIMIT 1 +238 val_238