Index: cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java =================================================================== --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (revision 901511) +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (working copy) @@ -18,25 +18,37 @@ package org.apache.hadoop.hive.cli; -import jline.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Vector; -import java.io.*; -import java.util.*; +import jline.ArgumentCompletor; +import jline.ConsoleReader; +import jline.History; +import jline.SimpleCompletor; -import org.apache.hadoop.fs.FsShell; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; +import org.apache.hadoop.hive.ql.processors.CommandProcessor; +import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.ql.processors.CommandProcessor; -import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.shims.ShimLoader; public class CliDriver { @@ -44,26 +56,27 @@ public final static String prompt = "hive"; public final static String prompt2 = " "; // when ';' is not yet seen - private LogHelper console; - private Configuration conf; + private final LogHelper console; + private final Configuration conf; public CliDriver() { SessionState ss = SessionState.get(); - conf = (ss != null) ? ss.getConf() : new Configuration (); + conf = (ss != null) ? ss.getConf() : new Configuration(); Log LOG = LogFactory.getLog("CliDriver"); console = new LogHelper(LOG); } - + public int processCmd(String cmd) { SessionState ss = SessionState.get(); - + String cmd_trimmed = cmd.trim(); String[] tokens = cmd_trimmed.split("\\s+"); String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim(); int ret = 0; - - if (cmd_trimmed.toLowerCase().equals("quit") || cmd_trimmed.toLowerCase().equals("exit")) { + if (cmd_trimmed.toLowerCase().equals("quit") + || cmd_trimmed.toLowerCase().equals("exit")) { + // if we have come this far - either the previous commands // are all successful or this is command line. in either case // this counts as a successful run @@ -73,49 +86,53 @@ String shell_cmd = cmd_trimmed.substring(1); - //shell_cmd = "/bin/bash -c \'" + shell_cmd + "\'"; + // shell_cmd = "/bin/bash -c \'" + shell_cmd + "\'"; try { Process executor = Runtime.getRuntime().exec(shell_cmd); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, ss.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, ss.err); - + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), + null, ss.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), + null, ss.err); + outPrinter.start(); errPrinter.start(); - + ret = executor.waitFor(); if (ret != 0) { console.printError("Command failed with exit code = " + ret); } - } - catch (Exception e) { - console.printError("Exception raised from Shell command " + e.getLocalizedMessage(), - org.apache.hadoop.util.StringUtils.stringifyException(e)); + } catch (Exception e) { + console.printError("Exception raised from Shell command " + + e.getLocalizedMessage(), org.apache.hadoop.util.StringUtils + .stringifyException(e)); ret = 1; } } else if (tokens[0].toLowerCase().equals("list")) { SessionState.ResourceType t; - if(tokens.length < 2 || (t = SessionState.find_resource_type(tokens[1])) == null) { - console.printError("Usage: list [" + - StringUtils.join(SessionState.ResourceType.values(),"|") + - "] [ []*]" ); + if (tokens.length < 2 + || (t = SessionState.find_resource_type(tokens[1])) == null) { + console.printError("Usage: list [" + + StringUtils.join(SessionState.ResourceType.values(), "|") + + "] [ []*]"); ret = 1; } else { List filter = null; - if(tokens.length >=3) { - System.arraycopy(tokens, 2, tokens, 0, tokens.length-2); + if (tokens.length >= 3) { + System.arraycopy(tokens, 2, tokens, 0, tokens.length - 2); filter = Arrays.asList(tokens); } Set s = ss.list_resource(t, filter); - if(s != null && !s.isEmpty()) + if (s != null && !s.isEmpty()) { ss.out.println(StringUtils.join(s, "\n")); + } } } else { CommandProcessor proc = CommandProcessorFactory.get(tokens[0]); - if(proc != null) { - if(proc instanceof Driver) { + if (proc != null) { + if (proc instanceof Driver) { Driver qp = (Driver) proc; PrintStream out = ss.out; long start = System.currentTimeMillis(); @@ -125,11 +142,11 @@ qp.close(); return ret; } - + Vector res = new Vector(); try { while (qp.getResults(res)) { - for (String r:res) { + for (String r : res) { out.println(r); } res.clear(); @@ -138,11 +155,12 @@ } } } catch (IOException e) { - console.printError("Failed with exception " + e.getClass().getName() + ":" + e.getMessage(), - "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); + console.printError("Failed with exception " + + e.getClass().getName() + ":" + e.getMessage(), "\n" + + org.apache.hadoop.util.StringUtils.stringifyException(e)); ret = 1; } - + int cret = qp.close(); if (ret == 0) { ret = cret; @@ -150,7 +168,7 @@ long end = System.currentTimeMillis(); if (end > start) { - double timeTaken = (double)(end-start)/1000.0; + double timeTaken = (end - start) / 1000.0; console.printInfo("Time taken: " + timeTaken + " seconds", null); } @@ -165,24 +183,26 @@ public int processLine(String line) { int lastRet = 0, ret = 0; - - String command=""; - for(String oneCmd: line.split(";")) { - - if (StringUtils.endsWith(oneCmd, "\\")){ - command+=StringUtils.chop(oneCmd)+";"; + + String command = ""; + for (String oneCmd : line.split(";")) { + + if (StringUtils.endsWith(oneCmd, "\\")) { + command += StringUtils.chop(oneCmd) + ";"; continue; } else { - command+=oneCmd; + command += oneCmd; } - if(StringUtils.isBlank(command)) + if (StringUtils.isBlank(command)) { continue; - + } + ret = processCmd(command); - command=""; + command = ""; lastRet = ret; - boolean ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS); - if(ret != 0 && !ignoreErrors) { + boolean ignoreErrors = HiveConf.getBoolVar(conf, + HiveConf.ConfVars.CLIIGNOREERRORS); + if (ret != 0 && !ignoreErrors) { return ret; } } @@ -193,7 +213,7 @@ String line; StringBuffer qsb = new StringBuffer(); - while((line = r.readLine()) != null) { + while ((line = r.readLine()) != null) { qsb.append(line + "\n"); } @@ -203,15 +223,16 @@ public static void main(String[] args) throws Exception { OptionsProcessor oproc = new OptionsProcessor(); - if(! oproc.process_stage1(args)) { + if (!oproc.process_stage1(args)) { System.exit(1); } - // NOTE: It is critical to do this here so that log4j is reinitialized before + // NOTE: It is critical to do this here so that log4j is reinitialized + // before // any of the other core hive classes are loaded SessionState.initHiveLog4j(); - CliSessionState ss = new CliSessionState (new HiveConf(SessionState.class)); + CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); ss.in = System.in; try { ss.out = new PrintStream(System.out, true, "UTF-8"); @@ -220,24 +241,25 @@ System.exit(3); } - - if(! oproc.process_stage2(ss)) { + if (!oproc.process_stage2(ss)) { System.exit(2); } // set all properties specified via command line HiveConf conf = ss.getConf(); - for(Map.Entry item: ss.cmdProperties.entrySet()) { + for (Map.Entry item : ss.cmdProperties.entrySet()) { conf.set((String) item.getKey(), (String) item.getValue()); } - - if(!ShimLoader.getHadoopShims().usesJobShell()) { - // hadoop-20 and above - we need to augment classpath using hiveconf components + + if (!ShimLoader.getHadoopShims().usesJobShell()) { + // hadoop-20 and above - we need to augment classpath using hiveconf + // components // see also: code in ExecDriver.java ClassLoader loader = conf.getClassLoader(); String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); if (StringUtils.isNotBlank(auxJars)) { - loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ",")); + loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, + ",")); } conf.setClassLoader(loader); Thread.currentThread().setContextClassLoader(loader); @@ -245,44 +267,46 @@ SessionState.start(ss); - CliDriver cli = new CliDriver (); + CliDriver cli = new CliDriver(); - if(ss.execString != null) { + if (ss.execString != null) { System.exit(cli.processLine(ss.execString)); } try { - if(ss.fileName != null) { - System.exit(cli.processReader(new BufferedReader(new FileReader(ss.fileName)))); + if (ss.fileName != null) { + System.exit(cli.processReader(new BufferedReader(new FileReader( + ss.fileName)))); } } catch (FileNotFoundException e) { - System.err.println("Could not open input file for reading. ("+e.getMessage()+")"); + System.err.println("Could not open input file for reading. (" + + e.getMessage() + ")"); System.exit(3); } ConsoleReader reader = new ConsoleReader(); reader.setBellEnabled(false); - //reader.setDebug(new PrintWriter(new FileWriter("writer.debug", true))); + // reader.setDebug(new PrintWriter(new FileWriter("writer.debug", true))); List completors = new LinkedList(); - completors.add(new SimpleCompletor(new String[] { "set", "from", - "create", "load", - "describe", "quit", "exit" })); + completors.add(new SimpleCompletor(new String[] { "set", "from", "create", + "load", "describe", "quit", "exit" })); reader.addCompletor(new ArgumentCompletor(completors)); - + String line; final String HISTORYFILE = ".hivehistory"; - String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE; + String historyFile = System.getProperty("user.home") + File.separator + + HISTORYFILE; reader.setHistory(new History(new File(historyFile))); int ret = 0; String prefix = ""; String curPrompt = prompt; - while ((line = reader.readLine(curPrompt+"> ")) != null) { + while ((line = reader.readLine(curPrompt + "> ")) != null) { if (!prefix.equals("")) { prefix += '\n'; } - if(line.trim().endsWith(";") && !line.trim().endsWith("\\;")) { + if (line.trim().endsWith(";") && !line.trim().endsWith("\\;")) { line = prefix + line; ret = cli.processLine(line); prefix = ""; Index: cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java =================================================================== --- cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java (revision 901511) +++ cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java (working copy) @@ -18,160 +18,162 @@ package org.apache.hadoop.hive.cli; -import java.io.*; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; -import org.apache.commons.cli2.*; +import org.apache.commons.cli2.Argument; +import org.apache.commons.cli2.CommandLine; +import org.apache.commons.cli2.Group; +import org.apache.commons.cli2.Option; +import org.apache.commons.cli2.OptionException; +import org.apache.commons.cli2.WriteableCommandLine; import org.apache.commons.cli2.builder.ArgumentBuilder; import org.apache.commons.cli2.builder.DefaultOptionBuilder; import org.apache.commons.cli2.builder.GroupBuilder; import org.apache.commons.cli2.commandline.Parser; import org.apache.commons.cli2.option.PropertyOption; import org.apache.commons.cli2.resource.ResourceConstants; -import org.apache.commons.cli2.OptionException; -import org.apache.commons.logging.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.conf.HiveConf; - public class OptionsProcessor { - protected static final Log l4j = LogFactory.getLog(OptionsProcessor.class.getName()); + protected static final Log l4j = LogFactory.getLog(OptionsProcessor.class + .getName()); - private Parser parser = new Parser(); - private Option confOptions, isSilentOption, execOption, fileOption, isHelpOption; + private final Parser parser = new Parser(); + private final Option confOptions, isSilentOption, execOption, fileOption, + isHelpOption; - /** - * shameless cloned from hadoop streaming - * take in multiple -hiveconf x=y parameters + /** + * shameless cloned from hadoop streaming take in multiple -hiveconf x=y + * parameters */ - class MultiPropertyOption extends PropertyOption{ - private String optionString; - MultiPropertyOption(){ - super(); + class MultiPropertyOption extends PropertyOption { + private String optionString; + + MultiPropertyOption() { + super(); } - - MultiPropertyOption(final String optionString, - final String description, - final int id){ - super(optionString, description, id); + + MultiPropertyOption(final String optionString, final String description, + final int id) { + super(optionString, description, id); this.optionString = optionString; } + @Override public boolean canProcess(final WriteableCommandLine commandLine, - final String argument) { + final String argument) { boolean ret = (argument != null) && argument.startsWith(optionString); - + return ret; - } + } + @Override public void process(final WriteableCommandLine commandLine, - final ListIterator arguments) throws OptionException { + final ListIterator arguments) throws OptionException { final String arg = (String) arguments.next(); if (!canProcess(commandLine, arg)) { - throw new OptionException(this, - ResourceConstants.UNEXPECTED_TOKEN, arg); + throw new OptionException(this, ResourceConstants.UNEXPECTED_TOKEN, arg); } - - ArrayList properties = new ArrayList(); - String next = ""; - while(arguments.hasNext()){ + + ArrayList properties = new ArrayList(); + String next = ""; + while (arguments.hasNext()) { next = (String) arguments.next(); - if (!next.startsWith("-")){ - - if(next.indexOf("=") == -1) { + if (!next.startsWith("-")) { + + if (next.indexOf("=") == -1) { throw new OptionException(this, ResourceConstants.UNEXPECTED_TOKEN, - "argument: '" + next + "' is not of the form x=y"); + "argument: '" + next + "' is not of the form x=y"); } properties.add(next); - }else{ + } else { arguments.previous(); - break; + break; } - } + } // add to any existing values (support specifying args multiple times) - List oldVal = (List)commandLine.getValue(this); - if (oldVal == null){ + List oldVal = (List) commandLine.getValue(this); + if (oldVal == null) { commandLine.addValue(this, properties); - } else{ - oldVal.addAll(properties); + } else { + oldVal.addAll(properties); } } } - private Option createBoolOption(DefaultOptionBuilder builder, String longName, - String shortName, String desc){ + private Option createBoolOption(DefaultOptionBuilder builder, + String longName, String shortName, String desc) { builder.reset(); - if(longName == null) { + if (longName == null) { return builder.withShortName(shortName).withDescription(desc).create(); } else { - return builder.withShortName(shortName).withLongName(longName).withDescription(desc).create(); + return builder.withShortName(shortName).withLongName(longName) + .withDescription(desc).create(); } } - - private Option createOptionWithArg(DefaultOptionBuilder builder, String longName, - String shortName, String desc, Argument arg) { + private Option createOptionWithArg(DefaultOptionBuilder builder, + String longName, String shortName, String desc, Argument arg) { builder.reset(); - DefaultOptionBuilder dob = - builder.withShortName(shortName). - withArgument(arg). - withDescription(desc); + DefaultOptionBuilder dob = builder.withShortName(shortName).withArgument( + arg).withDescription(desc); - if(longName != null) + if (longName != null) { dob = dob.withLongName(longName); + } return dob.create(); } - public OptionsProcessor() { - DefaultOptionBuilder builder = - new DefaultOptionBuilder("-","-", false); + DefaultOptionBuilder builder = new DefaultOptionBuilder("-", "-", false); ArgumentBuilder argBuilder = new ArgumentBuilder(); - - //-e - execOption = createOptionWithArg(builder, "exec", "e", "execute the following command", - argBuilder.withMinimum(1).withMaximum(1).create()); + // -e + execOption = createOptionWithArg(builder, "exec", "e", + "execute the following command", argBuilder.withMinimum(1).withMaximum( + 1).create()); - //-f - fileOption = createOptionWithArg(builder, "file", "f", "execute commands from the following file", - argBuilder.withMinimum(1).withMaximum(1).create()); + // -f + fileOption = createOptionWithArg(builder, "file", "f", + "execute commands from the following file", argBuilder.withMinimum(1) + .withMaximum(1).create()); // -S isSilentOption = createBoolOption(builder, "silent", "S", "silent mode"); // -help isHelpOption = createBoolOption(builder, "help", "h", "help"); - + // -hiveconf var=val - confOptions = new MultiPropertyOption("-hiveconf", "(n=v) Optional. Add or override Hive/Hadoop properties.", 'D'); + confOptions = new MultiPropertyOption("-hiveconf", + "(n=v) Optional. Add or override Hive/Hadoop properties.", 'D'); new PropertyOption(); - Group allOptions = new GroupBuilder(). - withOption(confOptions). - withOption(isSilentOption). - withOption(isHelpOption). - withOption(execOption). - withOption(fileOption). - create(); + Group allOptions = new GroupBuilder().withOption(confOptions).withOption( + isSilentOption).withOption(isHelpOption).withOption(execOption) + .withOption(fileOption).create(); parser.setGroup(allOptions); } private CommandLine cmdLine; - - public boolean process_stage1(String [] argv) { + + public boolean process_stage1(String[] argv) { try { cmdLine = parser.parse(argv); - List hiveConfArgs = (List)cmdLine.getValue(confOptions); - if (null != hiveConfArgs){ - for(String s : hiveConfArgs){ - String []parts = s.split("=", 2); + List hiveConfArgs = (List) cmdLine.getValue(confOptions); + if (null != hiveConfArgs) { + for (String s : hiveConfArgs) { + String[] parts = s.split("=", 2); System.setProperty(parts[0], parts[1]); } } @@ -182,29 +184,28 @@ return true; } - public boolean process_stage2(CliSessionState ss) { - HiveConf hconf = ss.getConf(); - //-S + ss.getConf(); + // -S ss.setIsSilent(cmdLine.hasOption(isSilentOption)); - //-e + // -e ss.execString = (String) cmdLine.getValue(execOption); - //-f + // -f ss.fileName = (String) cmdLine.getValue(fileOption); // -h if (cmdLine.hasOption(isHelpOption)) { printUsage(null); return false; } - if(ss.execString != null && ss.fileName != null) { + if (ss.execString != null && ss.fileName != null) { printUsage("-e and -f option cannot be specified simultaneously"); return false; } - List hiveConfArgs = (List)cmdLine.getValue(confOptions); - if (null != hiveConfArgs){ - for(String s : hiveConfArgs){ - String []parts = s.split("=", 2); + List hiveConfArgs = (List) cmdLine.getValue(confOptions); + if (null != hiveConfArgs) { + for (String s : hiveConfArgs) { + String[] parts = s.split("=", 2); ss.cmdProperties.setProperty(parts[0], parts[1]); } } @@ -212,20 +213,23 @@ return true; } - public void printUsage (String error) { + public void printUsage(String error) { if (error != null) { System.err.println("Invalid arguments: " + error); } System.err.println(""); - System.err.println("Usage: hive [--config confdir] [-hiveconf x=y]* [<-f filename>|<-e query-string>] [-S]"); + System.err + .println("Usage: hive [--config confdir] [-hiveconf x=y]* [<-f filename>|<-e query-string>] [-S]"); System.err.println(""); - System.err.println(" -e 'quoted query string' Sql from command line"); + System.err.println(" -e 'quoted query string' Sql from command line"); System.err.println(" -f Sql from files"); - System.err.println(" -S Silent mode in interactive shell"); + System.err + .println(" -S Silent mode in interactive shell"); System.err.println(""); - System.err.println("-e and -f cannot be specified together. In the absence of these"); + System.err + .println("-e and -f cannot be specified together. In the absence of these"); System.err.println("options, interactive shell is started"); System.err.println(""); - + } } Index: cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java =================================================================== --- cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java (revision 901511) +++ cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java (working copy) @@ -20,14 +20,13 @@ import java.util.Properties; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.session.SessionState; public class CliSessionState extends SessionState { /** * -e option if any that the session has been invoked with - */ + */ public String execString; /** @@ -40,12 +39,11 @@ */ public Properties cmdProperties = new Properties(); - public CliSessionState() { super(); } - public CliSessionState (HiveConf conf) { + public CliSessionState(HiveConf conf) { super(conf); }