diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 479fa46..81f9787 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1813,6 +1813,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", "", "A comma separated list of hooks which implement HiveDriverRunHook. Will be run at the beginning " + "and end of Driver.run, these will be run in the order specified."), + HIVE_PRE_PARSE_HOOKS("hive.exec.driver.preparse.hooks", "", + "A comma separated list of hooks which implement PreParseHook. Will be run after variable " + + "substitution of Driver.compile, these will be run in the order specified."), HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format", null, "The data format to use for DDL output. One of \"text\" (for human\n" + "readable text) or \"json\" (for a json object)."), diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 29e6315..bd6b495 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hive.ql.hooks.HookUtils; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; +import org.apache.hadoop.hive.ql.hooks.PreParseHook; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; @@ -375,6 +376,16 @@ public int compile(String command, boolean resetTaskIds) { } }).substitute(conf, command); + try { + List preParseHooks = getHooks(HiveConf.ConfVars.HIVE_PRE_PARSE_HOOKS, + PreParseHook.class); + for (PreParseHook preParseHook : preParseHooks) { + command = preParseHook.getCustomCommand(ctx, command); + } + } catch (Exception e) { + LOG.warn("WARNING! Query command could not be customized by pre parse hook. " + e); + } + String queryStr = command; try { diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/PreParseHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/PreParseHook.java new file mode 100644 index 0000000..3edf856 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/PreParseHook.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * PreParseHook allows Hive QL Statements to be extended with custom + * logic. A list of such hooks can be configured to be + * called after variable substitution and before query parse, + * allowing you to change/customize QL statement before hive parse + */ +public interface PreParseHook extends Hook { + /** + * Invoked before Hive performs parse analysis on + * a statement. The implementation may inspect the SQL command and + * change/replace/substitute text and creating a new SQL command + * to be used as hive QL + * + * @param context context information for statement analysis + * + * @param command original QL command + * + * @return replacement command (a new command QL statement customized + * based on original QL command and hook logic) + */ + public String getCustomCommand(Context context, String command) + throws Exception; + +} + diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/TestPreParseHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/TestPreParseHook.java new file mode 100644 index 0000000..750ba26 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/TestPreParseHook.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.AfterClass; +import org.junit.Test; + +public class TestPreParseHook { + + @BeforeClass + public static void onetimeSetup() throws Exception { + HiveConf conf = new HiveConf(TestPreParseHook.class); + Driver driver = createDriver(conf); + int ret = driver.run("create table t1(i int)").getResponseCode(); + assertEquals("Checking command success", 0, ret); + } + + @AfterClass + public static void onetimeTeardown() throws Exception { + HiveConf conf = new HiveConf(TestPreParseHook.class); + Driver driver = createDriver(conf); + driver.run("drop table t1"); + } + + @Before + public void setup() { + } + + @Test + public void testWithoutHook() throws Exception { + HiveConf conf = new HiveConf(TestPreParseHook.class); + Driver driver = createDriver(conf); + int ret = driver.compile("SELECT 'XXX' from t1"); + assertEquals("Checking command success", 0, ret); + } + + @Test + public void testBrokenHook() throws Exception { + HiveConf conf = new HiveConf(TestPreParseHook.class); + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_PRE_PARSE_HOOKS, + BrokenPreParseHook.class.getName()); + Driver driver = createDriver(conf); + int ret = driver.compile("SELECT 'XXX' from t1"); + assertEquals("Checking command success", 0, ret); + } + + @Test + public void testPreParseHook() throws Exception { + HiveConf conf = new HiveConf(TestPreParseHook.class); + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_PRE_PARSE_HOOKS, + SimpleCustomSQL.class.getName()); + Driver driver = createDriver(conf); + int ret = driver.compile("SLCT 'XXX' from t1"); + assertEquals("Checking command success", 0, ret); + } + + public static class SimpleCustomSQL implements PreParseHook { + + public String getCustomCommand(Context context, String command) + throws Exception { + return command.replaceAll("SLCT", "SELECT"); + } + + } + + public static class BrokenPreParseHook implements PreParseHook { + + public String getCustomCommand(Context context, String command) + throws Exception { + throw new Exception("broken hook"); + } + + } + + private static Driver createDriver(HiveConf conf) { + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + SessionState.start(conf); + Driver driver = new Driver(conf); + driver.init(); + return driver; + } + +}