diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index d7a9b0e..7dc2fb2 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -42,9 +42,7 @@ import jline.SimpleCompletor; import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.HiveInterruptUtils; @@ -90,13 +88,17 @@ public static final String HIVERCFILE = ".hiverc"; private final LogHelper console; - private Configuration conf; + private final HiveConf conf; + private final Driver driver; public CliDriver() { - SessionState ss = SessionState.get(); - conf = (ss != null) ? ss.getConf() : new Configuration(); - Log LOG = LogFactory.getLog("CliDriver"); - console = new LogHelper(LOG); + this(SessionState.get() != null ? SessionState.get().getConf() : new HiveConf()); + } + + public CliDriver(HiveConf conf) { + this.conf = conf; + console = new LogHelper(LogFactory.getLog("CliDriver")); + driver = new Driver(conf); } public int processCmd(String cmd) { @@ -195,7 +197,11 @@ public int processCmd(String cmd) { } } else { // local mode try { - CommandProcessor proc = CommandProcessorFactory.get(tokens, (HiveConf) conf); + CommandProcessor proc = CommandProcessorFactory.getForHiveCommand(tokens); + if (proc == null) { + proc = driver; + } + proc.init(); ret = processLocalCmd(cmd, proc, ss); } catch (SQLException e) { console.printError("Failed processing command " + tokens[0] + " " + e.getLocalizedMessage(), @@ -208,14 +214,6 @@ public int processCmd(String cmd) { } /** - * For testing purposes to inject Configuration dependency - * @param conf to replace default - */ - void setConf(Configuration conf) { - this.conf = conf; - } - - /** * Extract and clean up the first command in the input. */ private String getFirstCmd(String cmd, int length) { @@ -293,13 +291,10 @@ int processLocalCmd(String cmd, CommandProcessor proc, CliSessionState ss) { console.printInfo("Time taken: " + timeTaken + " seconds" + (counter == 0 ? "" : ", Fetched: " + counter + " row(s)")); } else { - String firstToken = tokenizeCmd(cmd.trim())[0]; - String cmd_1 = getFirstCmd(cmd.trim(), firstToken.length()); - if (ss.getIsVerbose()) { - ss.out.println(firstToken + " " + cmd_1); + ss.out.println(cmd); } - CommandProcessorResponse res = proc.run(cmd_1); + CommandProcessorResponse res = proc.run(cmd); if (res.getResponseCode() != 0) { ss.out.println("Query returned non-zero code: " + res.getResponseCode() + ", cause: " + res.getErrorMessage()); @@ -415,11 +410,9 @@ public void handle(Signal signal) { lastRet = ret; boolean ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS); if (ret != 0 && !ignoreErrors) { - CommandProcessorFactory.clean((HiveConf) conf); return ret; } } - CommandProcessorFactory.clean((HiveConf) conf); return lastRet; } finally { // Once we are done processing the line, restore the old handler diff --git cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java index 63668bc..5a3ffa3 100644 --- cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java +++ cli/src/test/org/apache/hadoop/hive/cli/TestCliDriverMethods.java @@ -48,7 +48,6 @@ import junit.framework.TestCase; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -114,14 +113,12 @@ public void testThatCliDriverPrintsNoHeaderForCommandsWithNoSchema() * won't actually be thrown */ private PrintStream headerPrintingTestDriver(Schema mockSchema) throws CommandNeedRetryException { - CliDriver cliDriver = new CliDriver(); - - // We want the driver to try to print the header... - - Configuration conf = mock(Configuration.class); + HiveConf conf = mock(HiveConf.class); when(conf.getBoolean(eq(ConfVars.HIVE_CLI_PRINT_HEADER.varname), anyBoolean())) .thenReturn(true); - cliDriver.setConf(conf); + CliDriver cliDriver = new CliDriver(conf); + + // We want the driver to try to print the header... Driver proc = mock(Driver.class); diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7a56cf2..19dd798 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1741,6 +1741,9 @@ HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile", "Comma separated list of non-SQL Hive commands users are authorized to execute"), + HIVE_COMMAND_PROCESSORS("hive.command.processors", "", + "Comma separated list of class names, implementing org.apache.hadoop.hive.ql.processors.HiveCommand"), + HIVE_SERVER2_SESSION_CHECK_INTERVAL("hive.server2.session.check.interval", "0ms", new TimeValidator(TimeUnit.MILLISECONDS, 3000l, true, null, false), "The check interval for session/operation timeout, which can be disabled by setting to zero or negative value."), diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/HCatCli.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/HCatCli.java index f8d69cc..321b8e5 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/HCatCli.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/HCatCli.java @@ -280,10 +280,16 @@ private static int processCmd(String cmd) { cmd = cmd.trim(); String firstToken = cmd.split("\\s+")[0].trim(); - if (firstToken.equalsIgnoreCase("set")) { - return new SetProcessor().run(cmd.substring(firstToken.length()).trim()).getResponseCode(); - } else if (firstToken.equalsIgnoreCase("dfs")) { - return new DfsProcessor(ss.getConf()).run(cmd.substring(firstToken.length()).trim()).getResponseCode(); + try { + if (firstToken.equalsIgnoreCase("set")) { + return new SetProcessor().run(cmd).getResponseCode(); + } else if (firstToken.equalsIgnoreCase("dfs")) { + return new DfsProcessor().run(cmd).getResponseCode(); + } + } catch (CommandNeedRetryException e) { + ss.err.println("Failed with exception " + e.getClass().getName() + ":" + + e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); + return 1; } HCatDriver driver = new HCatDriver(); diff --git hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java index 0ad8f89..0279c421 100644 --- hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java +++ hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java @@ -330,12 +330,9 @@ public void runQuery() { queryRet = new ArrayList(queries.size()); for (int i = 0; i < queries.size(); i++) { String cmd = queries.get(i); - String cmd_trimmed = cmd.trim(); - String[] tokens = cmd_trimmed.split("\\s+"); - String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim(); CommandProcessor proc = null; try { - proc = CommandProcessorFactory.get(tokens[0]); + proc = CommandProcessorFactory.get(cmd); } catch (SQLException e) { l4j.error(getSessionName() + " error processing " + cmd, e); } @@ -378,7 +375,7 @@ public void runQuery() { } } else { try { - queryRet.add(Integer.valueOf(proc.run(cmd_1).getResponseCode())); + queryRet.add(Integer.valueOf(proc.run(cmd).getResponseCode())); } catch (CommandNeedRetryException e) { // this should never happen if there is no bug l4j.error(getSessionName() + " Exception when executing", e); diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index ca55f17..741e801 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -284,16 +284,23 @@ public Driver(HiveConf conf) { } public Driver(HiveConf conf, String userName) { - this(conf); + this.conf = conf; this.userName = userName; } public Driver() { - if (SessionState.get() != null) { - conf = SessionState.get().getConf(); + SessionState session = SessionState.get(); + if (session != null) { + conf = session.getConf(); + userName = session.getUserName(); } } + public void init(HiveConf conf, String userName) { + this.conf = conf; + this.userName = userName; + } + /** * Compile a new query. Any currently-planned query associated with this Driver is discarded. * Do not reset id for inner queries(index, etc). Task ids are used for task identity check. diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java index 0558c53..ce27cf7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java @@ -32,18 +32,14 @@ * AddResourceProcessor. * */ -public class AddResourceProcessor implements CommandProcessor { +public class AddResourceProcessor extends SimpleProcessor { public static final Log LOG = LogFactory.getLog(AddResourceProcessor.class .getName()); public static final LogHelper console = new LogHelper(LOG); @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); command = new VariableSubstitution().substitute(ss.getConf(),command); String[] tokens = command.split("\\s+"); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java index 727f61f..e8985dfb2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java @@ -18,18 +18,20 @@ package org.apache.hadoop.hive.ql.processors; -import static org.apache.commons.lang.StringUtils.isBlank; - import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; /** * CommandProcessorFactory. @@ -37,84 +39,83 @@ */ public final class CommandProcessorFactory { + private static final Log LOG = LogFactory.getLog(CommandProcessorFactory.class.getName()); + + private static final List COMMANDS; + private static final Set WHITE_LISTS; + + static { + List commands = new ArrayList(); + // hive builtin first + commands.add(new NativeCommands.ADD()); + commands.add(new NativeCommands.COMPILE()); + commands.add(new NativeCommands.DELETE()); + commands.add(new NativeCommands.DFS()); + commands.add(new NativeCommands.LIST()); + commands.add(new NativeCommands.RELOAD()); + commands.add(new NativeCommands.RESET()); + commands.add(new NativeCommands.SET()); + + HiveConf hiveConf = new HiveConf(); + String var = hiveConf.getVar(HiveConf.ConfVars.HIVE_COMMAND_PROCESSORS); + for (String className : StringUtils.getTrimmedStrings(var)) { + if (!className.isEmpty()) { + try { + commands.add((HiveCommand) ReflectionUtils.newInstance( + Class.forName(className), hiveConf)); + } catch (Exception e) { + LOG.warn("Failed to register hive command " + className, e); + } + } + } + COMMANDS = Collections.unmodifiableList(commands); + + Set whiteLists = new HashSet(); + var = hiveConf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST); + for (String availableCommand : StringUtils.getTrimmedStrings(var)) { + whiteLists.add(availableCommand.toLowerCase()); + } + WHITE_LISTS = Collections.unmodifiableSet(whiteLists); + } + private CommandProcessorFactory() { // prevent instantiation } - private static final Map mapDrivers = Collections.synchronizedMap(new HashMap()); + @VisibleForTesting + static List getCommands() { + return COMMANDS; + } - public static CommandProcessor get(String cmd) - throws SQLException { - return get(new String[]{cmd}, null); + public static CommandProcessor get(String cmd) throws SQLException { + return get(cmd.trim().split("\\s+")); } - public static CommandProcessor getForHiveCommand(String[] cmd, HiveConf conf) - throws SQLException { - HiveCommand hiveCommand = HiveCommand.find(cmd); - if (hiveCommand == null || isBlank(cmd[0])) { - return null; - } - if (conf == null) { - conf = new HiveConf(); - } - Set availableCommands = new HashSet(); - for (String availableCommand : conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST).split(",")) { - availableCommands.add(availableCommand.toLowerCase().trim()); - } - if (!availableCommands.contains(cmd[0].trim().toLowerCase())) { - throw new SQLException("Insufficient privileges to execute " + cmd[0], "42000"); - } - switch (hiveCommand) { - case SET: - return new SetProcessor(); - case RESET: - return new ResetProcessor(); - case DFS: - SessionState ss = SessionState.get(); - return new DfsProcessor(ss.getConf()); - case ADD: - return new AddResourceProcessor(); - case LIST: - return new ListResourceProcessor(); - case DELETE: - return new DeleteResourceProcessor(); - case COMPILE: - return new CompileProcessor(); - case RELOAD: - return new ReloadProcessor(); - default: - throw new AssertionError("Unknown HiveCommand " + hiveCommand); + public static CommandProcessor get(String[] cmd) throws SQLException { + CommandProcessor result = getForHiveCommand(cmd); + if (result == null) { + result = new Driver(); } + return result; } - public static CommandProcessor get(String[] cmd, HiveConf conf) - throws SQLException { - CommandProcessor result = getForHiveCommand(cmd, conf); - if (result != null) { - return result; - } - if (isBlank(cmd[0])) { + public static CommandProcessor getForHiveCommand(String[] cmd) throws SQLException { + HiveCommand hiveCommand = cmd == null ? null : findCommand(cmd); + if (hiveCommand == null) { return null; - } else { - if (conf == null) { - return new Driver(); - } - Driver drv = mapDrivers.get(conf); - if (drv == null) { - drv = new Driver(); - mapDrivers.put(conf, drv); - } - drv.init(); - return drv; } + if (!WHITE_LISTS.contains(hiveCommand.getName().toLowerCase())) { + throw new SQLException("Insufficient privileges to execute " + cmd[0], "42000"); + } + return hiveCommand.getProcessor(cmd); } - public static void clean(HiveConf conf) { - Driver drv = mapDrivers.get(conf); - if (drv != null) { - drv.destroy(); + private static HiveCommand findCommand(String[] cmd) { + for (HiveCommand command : COMMANDS) { + if (command.accepts(cmd)) { + return command; + } } - - mapDrivers.remove(conf); + return null; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java index 25ce168..9cb4493 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java @@ -32,7 +32,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.VariableSubstitution; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; @@ -56,7 +55,7 @@ * each invocation of CompileProcessor. * */ -public class CompileProcessor implements CommandProcessor { +public class CompileProcessor extends SimpleProcessor { public static final Log LOG = LogFactory.getLog(CompileProcessor.class.getName()); public static final LogHelper console = new LogHelper(LOG); @@ -92,11 +91,6 @@ runCount = new AtomicInteger(0); } - @Override - public void init() { - //no init needed - } - /** * User supplies dynamic code in this format: * COMPILE ` some code here ` AS groovy NAMED something.groovy; @@ -107,7 +101,7 @@ public void init() { * @return CommandProcessorResponse with 0 for success and 1 for failure */ @Override - public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); this.command = command; diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java index 9052c82..799c578 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java @@ -32,17 +32,13 @@ * DeleteResourceProcessor. * */ -public class DeleteResourceProcessor implements CommandProcessor { +public class DeleteResourceProcessor extends SimpleProcessor { public static final Log LOG = LogFactory.getLog(DeleteResourceProcessor.class.getName()); public static final LogHelper console = new LogHelper(LOG); @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); command = new VariableSubstitution().substitute(ss.getConf(),command); String[] tokens = command.split("\\s+"); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java index cc0414d..a7cf64f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java @@ -23,7 +23,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; @@ -36,35 +35,26 @@ * DfsProcessor. * */ -public class DfsProcessor implements CommandProcessor { +public class DfsProcessor extends SimpleProcessor { public static final Log LOG = LogFactory.getLog(DfsProcessor.class.getName()); public static final LogHelper console = new LogHelper(LOG); public static final String DFS_RESULT_HEADER = "DFS Output"; - private final FsShell dfs; - private final Schema dfsSchema; + private static final Schema SCHEMA; - public DfsProcessor(Configuration conf) { - this(conf, false); - } - - public DfsProcessor(Configuration conf, boolean addSchema) { - dfs = new FsShell(conf); - dfsSchema = new Schema(); - dfsSchema.addToFieldSchemas(new FieldSchema(DFS_RESULT_HEADER, "string", "")); - } - - @Override - public void init() { + static { + SCHEMA = new Schema(); + SCHEMA.addToFieldSchemas(new FieldSchema(DFS_RESULT_HEADER, "string", null)); } @Override - public CommandProcessorResponse run(String command) { + public CommandProcessorResponse runCommand(String command) { + SessionState ss = SessionState.get(); + FsShell dfs = new FsShell(ss.getConf()); try { - SessionState ss = SessionState.get(); command = new VariableSubstitution().substitute(ss.getConf(),command); String[] tokens = command.split("\\s+"); @@ -87,7 +77,7 @@ public CommandProcessorResponse run(String command) { } System.setOut(oldOut); - return new CommandProcessorResponse(ret, null, null, dfsSchema); + return new CommandProcessorResponse(ret, null, null, SCHEMA); } catch (Exception e) { console.printError("Exception raised from DFSShell.run " diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java index 27d8325..673c936 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java @@ -18,45 +18,20 @@ package org.apache.hadoop.hive.ql.processors; -import java.util.HashSet; -import java.util.Set; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; /* * HiveCommand is non-SQL statement such as setting a property or * adding a resource. **/ -public enum HiveCommand { - SET(), - RESET(), - DFS(), - ADD(), - LIST(), - RELOAD(), - DELETE(), - COMPILE(); - private static final Set COMMANDS = new HashSet(); - static { - for (HiveCommand command : HiveCommand.values()) { - COMMANDS.add(command.name()); - } - } - public static HiveCommand find(String[] command) { - if (null == command){ - return null; - } - String cmd = command[0]; - if (cmd != null) { - cmd = cmd.trim().toUpperCase(); - if (command.length > 1 && "role".equalsIgnoreCase(command[1])) { - // special handling for set role r1 statement - return null; - } else if(command.length > 1 && "from".equalsIgnoreCase(command[1])) { - //special handling for SQL "delete from where..." - return null; - } else if (COMMANDS.contains(cmd)) { - return HiveCommand.valueOf(cmd); - } - } - return null; - } +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface HiveCommand { + + String getName(); + + boolean accepts(String[] command); + + CommandProcessor getProcessor(String[] command); } diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/ListResourceProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/ListResourceProcessor.java index 989cb87..e6166da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/ListResourceProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/ListResourceProcessor.java @@ -21,7 +21,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.session.SessionState; import java.util.Arrays; @@ -32,7 +31,7 @@ import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultNullString; -public class ListResourceProcessor implements CommandProcessor { +public class ListResourceProcessor extends SimpleProcessor { private static final String LIST_COLUMN_NAME = "resource"; private static final Schema SCHEMA; @@ -44,11 +43,7 @@ } @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); String[] tokens = command.split("\\s+"); SessionState.ResourceType t; diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java new file mode 100644 index 0000000..bd08732 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/processors/NativeCommands.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.processors; + +import org.apache.commons.lang3.StringUtils; + +// hive builtin commands +public class NativeCommands { + + private abstract static class AbstractCommand implements HiveCommand { + @Override + public String getName() { return getClass().getSimpleName().toLowerCase(); } + @Override + public boolean accepts(String[] command) { + return !StringUtils.isEmpty(command[0]) && getName().equalsIgnoreCase(command[0]); + } + } + + public static class SET extends AbstractCommand { + @Override + public boolean accepts(String[] command) { + // special handling for set role r1 statement + return super.accepts(command) && + (command.length == 1 || !command[1].equalsIgnoreCase("role")); + } + @Override + public CommandProcessor getProcessor(String[] command) { return new SetProcessor(); } + } + + public static class RESET extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ResetProcessor(); } + } + + public static class DFS extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new DfsProcessor(); } + } + + public static class ADD extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new AddResourceProcessor(); } + } + + public static class LIST extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ListResourceProcessor(); } + } + + public static class RELOAD extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new ReloadProcessor(); } + } + + public static class DELETE extends AbstractCommand { + @Override + public boolean accepts(String[] command) { + //special handling for SQL "delete from
where..." + return super.accepts(command) && + (command.length == 1 || !command[1].equalsIgnoreCase("from")); + } + @Override + public CommandProcessor getProcessor(String[] command) { return new DeleteResourceProcessor(); } + } + + public static class COMPILE extends AbstractCommand { + @Override + public CommandProcessor getProcessor(String[] command) { return new CompileProcessor(); } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java index b84c9dd..cc6dd63 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/ReloadProcessor.java @@ -22,21 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.session.SessionState; /** * used for reload auxiliary and jars without restarting hive server2 */ -public class ReloadProcessor implements CommandProcessor{ +public class ReloadProcessor extends SimpleProcessor { private static final Log LOG = LogFactory.getLog(ReloadProcessor.class); @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); try { ss.reloadAuxJars(); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java index e67422b..6b17f3b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java @@ -21,18 +21,13 @@ import java.util.Arrays; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.session.SessionState; -public class ResetProcessor implements CommandProcessor { +public class ResetProcessor extends SimpleProcessor { @Override - public void init() { - } - - @Override - public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); CommandProcessorResponse authErrResp = diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index bc9254c..e3e078b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -40,7 +40,7 @@ * SetProcessor. * */ -public class SetProcessor implements CommandProcessor { +public class SetProcessor extends SimpleProcessor { private static final String prefix = "set: "; @@ -97,10 +97,6 @@ private void dumpOption(String s) { } } - @Override - public void init() { - } - public CommandProcessorResponse executeSetVariable(String varname, String varvalue) { try { return new CommandProcessorResponse(setVariable(varname, varvalue)); @@ -249,7 +245,7 @@ private CommandProcessorResponse createProcessorSuccessResponse() { } @Override - public CommandProcessorResponse run(String command) { + public CommandProcessorResponse runCommand(String command) { SessionState ss = SessionState.get(); String nwcmd = command.trim(); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SimpleProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SimpleProcessor.java new file mode 100644 index 0000000..9fa3dc6 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SimpleProcessor.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.processors; + +import org.apache.hadoop.hive.ql.CommandNeedRetryException; + +public abstract class SimpleProcessor implements CommandProcessor { + + @Override + public void init() { + } + + @Override + public CommandProcessorResponse run(String command) throws CommandNeedRetryException { + return runCommand(getCommandArgs(command)); + } + + private String getCommandArgs(String statement) { + statement = statement.trim(); + String[] tokens = statement.split("\\s"); + return statement.substring(tokens[0].length()).trim(); + } + + protected abstract CommandProcessorResponse runCommand(String command) + throws CommandNeedRetryException; +} diff --git ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java index ac5053a..7cda984 100644 --- ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java +++ ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java @@ -38,27 +38,27 @@ public void setUp() throws Exception { @Test public void testInvalidCommands() throws Exception { - Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null, conf)); - Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "}, conf)); - Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"}, conf)); - Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"}, conf)); + Assert.assertNull("Null should have returned null", CommandProcessorFactory.getForHiveCommand(null)); + Assert.assertNull("Blank should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{" "})); + Assert.assertNull("set role should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"set role"})); + Assert.assertNull("SQL should have returned null", CommandProcessorFactory.getForHiveCommand(new String[]{"SELECT * FROM TABLE"})); } @Test public void testAvailableCommands() throws Exception { SessionState.start(conf); - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); + Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd})); } - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name().toLowerCase(); - Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf)); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); + Assert.assertNotNull("Cmd " + cmd + " not return null", CommandProcessorFactory.getForHiveCommand(new String[]{cmd})); } conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), ""); - for (HiveCommand command : HiveCommand.values()) { - String cmd = command.name(); + for (HiveCommand command : CommandProcessorFactory.getCommands()) { + String cmd = command.getName(); try { - CommandProcessorFactory.getForHiveCommand(new String[]{cmd}, conf); + CommandProcessorFactory.getForHiveCommand(new String[]{cmd}); Assert.fail("Expected SQLException for " + cmd + " as available commands is empty"); } catch (SQLException e) { Assert.assertEquals("Insufficient privileges to execute " + cmd, e.getMessage()); diff --git service/src/java/org/apache/hadoop/hive/service/HiveServer.java service/src/java/org/apache/hadoop/hive/service/HiveServer.java index 27c7722..f81e219 100644 --- service/src/java/org/apache/hadoop/hive/service/HiveServer.java +++ service/src/java/org/apache/hadoop/hive/service/HiveServer.java @@ -170,10 +170,6 @@ public void execute(String cmd) throws HiveServerException, TException { HiveServerHandler.LOG.info("Running the query: " + cmd); SessionState session = SessionState.get(); - String cmd_trimmed = cmd.trim(); - String[] tokens = cmd_trimmed.split("\\s"); - String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim(); - int ret = 0; String errorMessage = ""; String SQLState = null; @@ -185,7 +181,7 @@ public void execute(String cmd) throws HiveServerException, TException { driver.close(); driver = null; } - CommandProcessor proc = CommandProcessorFactory.get(tokens[0]); + CommandProcessor proc = CommandProcessorFactory.get(cmd); if (proc != null) { if (proc instanceof Driver) { isHiveQuery = true; @@ -200,7 +196,7 @@ public void execute(String cmd) throws HiveServerException, TException { driver = null; // need to reset output for each non-Hive query setupSessionIO(session); - response = proc.run(cmd_1); + response = proc.run(cmd); } ret = response.getResponseCode(); diff --git service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java index 3f2de10..d9a9554 100644 --- service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; import org.apache.hive.service.cli.HiveSQLException; @@ -46,14 +47,14 @@ public static ExecuteStatementOperation newExecuteStatementOperation( HiveSession parentSession, String statement, Map confOverlay, boolean runAsync) throws HiveSQLException { String[] tokens = statement.trim().split("\\s+"); - CommandProcessor processor = null; + CommandProcessor processor; try { - processor = CommandProcessorFactory.getForHiveCommand(tokens, parentSession.getHiveConf()); + processor = CommandProcessorFactory.get(tokens); } catch (SQLException e) { throw new HiveSQLException(e.getMessage(), e.getSQLState(), e); } - if (processor == null) { - return new SQLOperation(parentSession, statement, confOverlay, runAsync); + if (processor instanceof Driver) { + return new SQLOperation(parentSession, statement, confOverlay, (Driver)processor, runAsync); } return new HiveCommandOperation(parentSession, statement, processor, confOverlay); } diff --git service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 8cabf7e..a958744 100644 --- service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -76,9 +76,10 @@ private boolean fetchStarted = false; public SQLOperation(HiveSession parentSession, String statement, Map confOverlay, boolean runInBackground) { + String> confOverlay, Driver driver, boolean runInBackground) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); + this.driver = driver; } /*** @@ -90,7 +91,7 @@ public void prepare(HiveConf sqlOperationConf) throws HiveSQLException { setState(OperationState.RUNNING); try { - driver = new Driver(sqlOperationConf, getParentSession().getUserName()); + driver.init(sqlOperationConf, getParentSession().getUserName()); // In Hive server mode, we are not able to retry in the FetchTask // case, when calling fetch queries since execute() has returned. // For now, we disable the test attempts.