diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index d7a9b0e..3aa4d79 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -42,9 +42,7 @@ import jline.SimpleCompletor; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.HiveInterruptUtils; @@ -90,13 +88,17 @@ public static final String HIVERCFILE = ".hiverc"; private final LogHelper console; - private Configuration conf; + private final HiveConf conf; + private final Driver driver; public CliDriver() { - SessionState ss = SessionState.get(); - conf = (ss != null) ? ss.getConf() : new Configuration(); - Log LOG = LogFactory.getLog("CliDriver"); - console = new LogHelper(LOG); + this(SessionState.get() != null ? SessionState.get().getConf() : new HiveConf()); + } + + public CliDriver(HiveConf conf) { + this.conf = conf; + console = new LogHelper(LogFactory.getLog("CliDriver")); + driver = new Driver(conf); } public int processCmd(String cmd) { @@ -195,7 +197,11 @@ public int processCmd(String cmd) { } } else { // local mode try { - CommandProcessor proc = CommandProcessorFactory.get(tokens, (HiveConf) conf); + CommandProcessor proc = CommandProcessorFactory.getForHiveCommand(tokens); + if (proc == null) { + proc = driver; + } + proc.init(); ret = processLocalCmd(cmd, proc, ss); } catch (SQLException e) { console.printError("Failed processing command " + tokens[0] + " " + e.getLocalizedMessage(), @@ -208,14 +214,6 @@ public int processCmd(String cmd) { } /** - * For testing purposes to inject Configuration dependency - * @param conf to replace default - */ - void setConf(Configuration conf) { - this.conf = conf; - } - - /** * Extract and clean up the first command in the input. */ private String getFirstCmd(String cmd, int length) { @@ -415,11 +413,9 @@ public void handle(Signal signal) { lastRet = ret; boolean ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS); if (ret != 0 && !ignoreErrors) { - CommandProcessorFactory.clean((HiveConf) conf); return ret; } } - CommandProcessorFactory.clean((HiveConf) conf); return lastRet; } finally { // Once we are done processing the line, restore the old handler diff --git cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index 63668bc..5a3ffa3 100644 --- cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -48,7 +48,6 @@ import junit.framework.TestCase; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -114,14 +113,12 @@ public void testThatCliDriverPrintsNoHeaderForCommandsWithNoSchema() * won't actually be thrown */ private PrintStream headerPrintingTestDriver(Schema mockSchema) throws CommandNeedRetryException { - CliDriver cliDriver = new CliDriver(); - - // We want the driver to try to print the header... - - Configuration conf = mock(Configuration.class); + HiveConf conf = mock(HiveConf.class); when(conf.getBoolean(eq(ConfVars.HIVE_CLI_PRINT_HEADER.varname), anyBoolean())) .thenReturn(true); - cliDriver.setConf(conf); + CliDriver cliDriver = new CliDriver(conf); + + // We want the driver to try to print the header... Driver proc = mock(Driver.class); diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index f9911ce..c7a612e 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1737,6 +1737,9 @@ HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile", "Comma separated list of non-SQL Hive commands users are authorized to execute"), + HIVE_COMMAND_PROCESSORS("hive.command.processors", "", + "Comma separated list of class names, implementing org.apache.hadoop.hive.ql.processors.HiveCommand"), + HIVE_SERVER2_SESSION_CHECK_INTERVAL("hive.server2.session.check.interval", "0ms", new TimeValidator(TimeUnit.MILLISECONDS, 3000l, true, null, false), "The check interval for session/operation timeout, which can be disabled by setting to zero or negative value."), diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index 727f61f..73eaf36 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -18,18 +18,20 @@ package org.apache.hadoop.hive.ql.processors; -import static org.apache.commons.lang.StringUtils.isBlank; - import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; /** * CommandProcessorFactory. @@ -37,84 +39,79 @@ */ public final class CommandProcessorFactory { + private static final Log LOG = LogFactory.getLog(CommandProcessorFactory.class.getName()); + + private static final List COMMANDS; + private static final Set WHITE_LISTS; + + static { + List commands = new ArrayList(); + // hive builtin first + commands.add(new NativeCommands.ADD()); + commands.add(new NativeCommands.COMPILE()); + commands.add(new NativeCommands.DELETE()); + commands.add(new NativeCommands.LIST()); + commands.add(new NativeCommands.RELOAD()); + commands.add(new NativeCommands.RESET()); + commands.add(new NativeCommands.SET()); + + HiveConf hiveConf = new HiveConf(); + String var = hiveConf.getVar(HiveConf.ConfVars.HIVE_COMMAND_PROCESSORS); + for (String className : StringUtils.getTrimmedStrings(var)) { + if (!className.isEmpty()) { + try { + commands.add((HiveCommand) ReflectionUtils.newInstance( + Class.forName(className), hiveConf)); + } catch (Exception e) { + LOG.warn("Failed to register hive command " + className, e); + } + } + } + COMMANDS = Collections.unmodifiableList(commands); + + Set whiteLists = new HashSet(); + var = hiveConf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST); + for (String availableCommand : StringUtils.getTrimmedStrings(var)) { + whiteLists.add(availableCommand.toLowerCase()); + } + WHITE_LISTS = Collections.unmodifiableSet(whiteLists); + } + private CommandProcessorFactory() { // prevent instantiation } - private static final Map mapDrivers = Collections.synchronizedMap(new HashMap()); + @VisibleForTesting + static List getCommands() { + return COMMANDS; + } - public static CommandProcessor get(String cmd) - throws SQLException { - return get(new String[]{cmd}, null); + public static CommandProcessor get(String cmd) throws SQLException { + CommandProcessor result = getForHiveCommand(new String[]{cmd}); + if (result == null) { + result = new Driver(); + } + return result; } - public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) + public static CommandProcessor getForHiveCommand(String[] cmd) throws SQLException { - HiveCommand hiveCommand = HiveCommand.find(cmd); - if (hiveCommand == null || isBlank(cmd[0])) { + HiveCommand hiveCommand = cmd == null ? null : findCommand(cmd); + if (hiveCommand == null) { return null; } - if (conf == null) { - conf = new HiveConf(); - } - Set availableCommands = new HashSet(); - for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST).split(",")) { - availableCommands.add(availableCommand.toLowerCase().trim()); - } - if (!availableCommands.contains(cmd[0].trim().toLowerCase())) { + if (!WHITE_LISTS.contains(hiveCommand.getName().toLowerCase())) { throw new SQLException("Insufficient privileges to execute " + cmd[0], "42000"); } - switch (hiveCommand) { - case SET: - return new SetProcessor(); - case RESET: - return new ResetProcessor(); - case DFS: - SessionState ss = SessionState.get(); - return new DfsProcessor(ss.getConf()); - case ADD: - return new AddResourceProcessor(); - case LIST: - return new ListResourceProcessor(); - case DELETE: - return new DeleteResourceProcessor(); - case COMPILE: - return new CompileProcessor(); - case RELOAD: - return new ReloadProcessor(); - default: - throw new AssertionError("Unknown HiveCommand " + hiveCommand); - } + return hiveCommand.getProcessor(cmd); } - public static CommandProcessor get(String[] cmd, HiveConf conf) - throws SQLException { - CommandProcessor result = getForHiveCommand(cmd, conf); - if (result != null) { - return result; - } - if (isBlank(cmd[0])) { - return null; - } else { - if (conf == null) { - return new Driver(); - } - Driver drv = mapDrivers.get(conf); - if (drv == null) { - drv = new Driver(); - mapDrivers.put(conf, drv); + private static HiveCommand findCommand(String[] cmd) { + for (HiveCommand command : COMMANDS) { + if (command.accepts(cmd)) { + return command; } - drv.init(); - return drv; - } - } - - public static void clean(HiveConf conf) { - Driver drv = mapDrivers.get(conf); - if (drv != null) { - drv.destroy(); } - - mapDrivers.remove(conf); + return null; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java index 27d8325..673c936 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java @@ -18,45 +18,20 @@ package org.apache.hadoop.hive.ql.processors; -import java.util.HashSet; -import java.util.Set; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; /* * HiveCommand is non-SQL statement such as setting a property or * adding a resource. **/ -public enum HiveCommand { - SET(), - RESET(), - DFS(), - ADD(), - LIST(), - RELOAD(), - DELETE(), - COMPILE(); - private static final Set COMMANDS = new HashSet(); - static { - for (HiveCommand command : HiveCommand.values()) { - COMMANDS.add(command.name()); - } - } - public static HiveCommand find(String[] command) { - if (null == command){ - return null; - } - String cmd = command[0]; - if (cmd != null) { - cmd = cmd.trim().toUpperCase(); - if (command.length > 1 && "role".equalsIgnoreCase(command[1])) { - // special handling for set role r1 statement - return null; - } else if(command.length > 1 && "from".equalsIgnoreCase(command[1])) { - //special handling for SQL "delete from where..." - return null; - } else if (COMMANDS.contains(cmd)) { - return HiveCommand.valueOf(cmd); - } - } - return null; - } +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface HiveCommand { + + String getName(); + + boolean accepts(String[] command); + + CommandProcessor getProcessor(String[] command); } diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java new file mode 100644 index 0000000..563fad0 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.processors; + +import org.apache.commons.lang3.StringUtils; + +// hive builtin commands +public class NativeCommands { + + private abstract static class AbstractCommand implements HiveCommand { + @Override + public String getName() { return getClass().getSimpleName().toLowerCase(); } + @Override + public boolean accepts(String[] command) { + return !StringUtils.isEmpty(command[0]) && getName().equalsIgnoreCase(command[0]); + } + } + + public static class SET extends AbstractCommand { + @Override + public boolean accepts(String[] command) { + // special handling for set role r1 statement + return super.accepts(command) && + (command.length == 1 || !command[1].equalsIgnoreCase("role")); + } + @Override + public CommandProcessor getProcessor(String[] command) { return new SetProcessor(); } + } + + public static class RESET extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ResetProcessor(); } + } + + public static class ADD extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new AddResourceProcessor(); } + } + + public static class LIST extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ListResourceProcessor(); } + } + + public static class RELOAD extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ReloadProcessor(); } + } + + public static class DELETE extends AbstractCommand { + @Override + public boolean accepts(String[] command) { + //special handling for SQL "delete from
where..." + return super.accepts(command) && + (command.length == 1 || !command[1].equalsIgnoreCase("from")); + } + @Override + public CommandProcessor getProcessor(String[] command) { return new DeleteResourceProcessor(); } + } + + public static class COMPILE extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new CompileProcessor(); } + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java index ac5053a..7cda984 100644 --- ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java +++ ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java @@ -38,27 +38,27 @@ public void setUp() throws Exception { @Test public void testInvalidCommands() throws Exception { - Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null, conf)); - Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); - Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); - Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null)); + Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "})); + Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"})); + Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"})); } @Test public void testAvailableCommands() throws Exception { SessionState.start(conf); - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); + Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd})); } - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name().toLowerCase(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); + Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd})); } conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), ""); - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name(); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); try { - CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf); + CommandProcessorFactory.getForHiveCommand(new String[]{cmd}); Assert.fail("Expected SQLException for " + cmd + " as available commands is empty"); } catch (SQLException e) { Assert.assertEquals("Insufficient privileges to execute " + cmd, e.getMessage()); diff --git service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java index 3f2de10..eb7088e 100644 --- service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java @@ -48,7 +48,7 @@ public static ExecuteStatementOperation newExecuteStatementOperation( String[] tokens = statement.trim().split("\\s+"); CommandProcessor processor = null; try { - processor = CommandProcessorFactory.getForHiveCommand(tokens, parentSession.getHiveConf()); + processor = CommandProcessorFactory.getForHiveCommand(tokens); } catch (SQLException e) { throw new HiveSQLException(e.getMessage(), e.getSQLState(), e); }