diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index 630ead4..e894f46 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -28,11 +28,9 @@ import java.io.EOFException; import java.io.File; import java.io.FileInputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; -import java.io.PrintWriter; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -79,7 +77,6 @@ import jline.console.history.History; import jline.console.history.FileHistory; -import jline.internal.Log; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.OptionBuilder; @@ -247,6 +244,8 @@ null), new ReflectiveCommandHandler(this, new String[] {"nullemptystring"}, new Completer[] {new BooleanCompleter()}), + new ReflectiveCommandHandler(this, new String[] {"addlocaljar"}, + null), }; @@ -1552,6 +1551,11 @@ boolean scanForDriver(String url) { return true; } + // find whether exists a local driver to accept the url + if (findLocalDriver(url) != null) { + return true; + } + return false; } catch (Exception e) { debug(e.toString()); @@ -1574,6 +1578,27 @@ private Driver findRegisteredDriver(String url) { return null; } + public Driver findLocalDriver(String url) throws Exception { + if(drivers == null){ + return null; + } + + for (Driver d : drivers) { + try { + String clazzName = d.getClass().getName(); + Driver driver = (Driver) Class.forName(clazzName, true, + Thread.currentThread().getContextClassLoader()).newInstance(); + if (driver.acceptsURL(url) && DatabaseConnection.isSupportedLocalDriver(driver)) { + return driver; + } + } catch (SQLException e) { + error(e); + throw new Exception(e); + } + } + return null; + } + Driver[] scanDrivers(String line) throws IOException { return scanDrivers(false); diff --git beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java index 065eab4..96a4787 100644 --- beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java +++ beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java @@ -52,9 +52,12 @@ package org.apache.hive.beeline; import jline.console.completer.StringsCompleter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.File; import java.io.IOException; +import java.io.FileInputStream; import java.net.JarURLConnection; import java.net.URL; import java.net.URLClassLoader; @@ -62,11 +65,15 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.Enumeration; import java.util.TreeSet; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipEntry; /** * the completer is original provided in JLine 0.9.94 and is being removed in 2.12. Add the @@ -74,6 +81,10 @@ */ public class ClassNameCompleter extends StringsCompleter { + private static final Log LOG = LogFactory.getLog(ClassNameCompleter.class.getName()); + public final static String clazzFileNameExtension = ".class"; + public final static String jarFileNameExtension = ".jar"; + public ClassNameCompleter(String... candidates) { super(candidates); } @@ -81,7 +92,7 @@ public ClassNameCompleter(String... candidates) { public static String[] getClassNames() throws IOException { Set urls = new HashSet(); - for (ClassLoader loader = ClassNameCompleter.class.getClassLoader(); loader != null; + for (ClassLoader loader = Thread.currentThread().getContextClassLoader(); loader != null; loader = loader.getParent()) { if (!(loader instanceof URLClassLoader)) { continue; @@ -97,7 +108,7 @@ public ClassNameCompleter(String... candidates) { for (int i = 0; i < systemClasses.length; i++) { URL classURL = systemClasses[i] - .getResource("/" + systemClasses[i].getName().replace('.', '/') + ".class"); + .getResource("/" + systemClasses[i].getName().replace('.', '/') + clazzFileNameExtension); if (classURL != null) { URLConnection uc = classURL.openConnection(); @@ -136,12 +147,15 @@ public ClassNameCompleter(String... candidates) { String name = entry.getName(); - if (!name.endsWith(".class")) { + if (name.endsWith(clazzFileNameExtension)) { /* only use class file*/ + classes.add(name); + } else if (isJarFile(name)) { + classes.addAll(getClassNamesFromJar(name)); + } else { continue; } - classes.add(name); } } @@ -151,8 +165,7 @@ public ClassNameCompleter(String... candidates) { for (Iterator i = classes.iterator(); i.hasNext(); ) { String name = (String) i.next(); - classNames.add(name.replace('/', '.'). - substring(0, name.length() - 6)); + classNames.add(name.replace('/', '.').substring(0, name.length() - 6)); } return (String[]) classNames.toArray(new String[classNames.size()]); @@ -173,7 +186,7 @@ private static Set getClassFiles(String root, Set holder, File directory, int[] continue; } else if (files[i].isDirectory()) { getClassFiles(root, holder, files[i], maxDirectories); - } else if (files[i].getName().endsWith(".class")) { + } else if (files[i].getName().endsWith(clazzFileNameExtension)) { holder.add(files[i].getAbsolutePath(). substring(root.length() + 1)); } @@ -181,4 +194,50 @@ private static Set getClassFiles(String root, Set holder, File directory, int[] return holder; } + + /** + * Get clazz names from a jar file path + * @param path specifies the jar file's path + * @return + */ + public static List getClassNamesFromJar(String path) { + List classNames = new ArrayList(); + ZipInputStream zip = null; + try { + zip = new ZipInputStream(new FileInputStream(path)); + ZipEntry entry = zip.getNextEntry(); + while (entry != null) { + if (!entry.isDirectory() && entry.getName().endsWith(clazzFileNameExtension)) { + StringBuilder className = new StringBuilder(); + for (String part : entry.getName().split("/")) { + if (className.length() != 0) { + className.append("."); + } + className.append(part); + if (part.endsWith(clazzFileNameExtension)) { + className.setLength(className.length() - clazzFileNameExtension.length()); + } + } + classNames.add(className.toString()); + } + entry = zip.getNextEntry(); + } + } catch (IOException e) { + LOG.error("Fail to parse the class name from the Jar file due to the exception:", e); + } finally { + if (zip != null) { + try { + zip.close(); + } catch (IOException e) { + LOG.error("Fail to close the file due to the exception:", e); + } + } + } + + return classNames; + } + + private static boolean isJarFile(String fileName) { + return fileName.endsWith(jarFileNameExtension); + } } diff --git beeline/src/java/org/apache/hive/beeline/Commands.java beeline/src/java/org/apache/hive/beeline/Commands.java index 291adba..d4ed8ef 100644 --- beeline/src/java/org/apache/hive/beeline/Commands.java +++ beeline/src/java/org/apache/hive/beeline/Commands.java @@ -25,12 +25,16 @@ import org.apache.hadoop.io.IOUtils; import java.io.BufferedReader; +import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -126,6 +130,29 @@ public boolean metadata(String cmd, String[] args) { return true; } + public boolean addlocaljar(String line){ + // If jar file is in the hdfs, it should be downloaded first. + String jarPath = arg1(line, "jar path"); + File p = new File(jarPath); + if (!p.exists()) { + beeLine.error("The jar file in the path " + jarPath + " can't be found!"); + return false; + } + + URLClassLoader classLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); + try { + beeLine.debug(jarPath + " is added to the local beeline."); + URLClassLoader newClassLoader = new URLClassLoader(new URL[]{p.toURL()}, classLoader); + + Thread.currentThread().setContextClassLoader(newClassLoader); + beeLine.setDrivers(Arrays.asList(beeLine.scanDrivers(false))); + } catch (MalformedURLException e) { + beeLine.error("Fail to add local jar due to the exception:" + e); + } catch (IOException e) { + beeLine.error("Fail to add local jar due to the exception:" + e); + } + return true; + } public boolean history(String line) { Iterator hist = beeLine.getConsoleReader().getHistory().entries(); diff --git beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java index 8ba0232..b9c07d2 100644 --- beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java +++ beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java @@ -25,9 +25,11 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DatabaseMetaData; +import java.sql.Driver; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -53,6 +55,10 @@ private Schema schema = null; private Completer sqlCompleter = null; + private final static String[] SUPPORTEDLOCALDRIVERS = new String[]{ + "com.mysql.jdbc.Driver", "org.postgresql.Driver" + }; + public boolean isClosed() { return (null == connection); } @@ -112,9 +118,9 @@ boolean connect() throws SQLException { return beeLine.error(cnfe); } - boolean foundDriver = false; + boolean isDriverRegistered = false; try { - foundDriver = DriverManager.getDriver(getUrl()) != null; + isDriverRegistered = DriverManager.getDriver(getUrl()) != null; } catch (Exception e) { } @@ -134,7 +140,13 @@ boolean connect() throws SQLException { info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue()); } - setConnection(DriverManager.getConnection(getUrl(), info)); + if(isDriverRegistered){ + // if the driver registered in the driver manager, get the connection via the driver manager + setConnection(DriverManager.getConnection(getUrl(), info)); + }else{ + beeLine.debug("Use the driver from local added jar file."); + setConnection(getConnectionFromLocalDriver(getUrl(), info)); + } setDatabaseMetaData(getConnection().getMetaData()); try { @@ -170,6 +182,40 @@ boolean connect() throws SQLException { return true; } + public Connection getConnectionFromLocalDriver(String url, Properties properties) { + Collection drivers = beeLine.getDrivers(); + for (Driver d : drivers) { + try { + if (d.acceptsURL(url)) { + String clazzName = d.getClass().getName(); + beeLine.debug("Driver name is " + clazzName); + Driver driver = + (Driver) Class.forName(clazzName, true, Thread.currentThread().getContextClassLoader()) + .newInstance(); + return driver.connect(url, properties); + } + } catch (SQLException e) { + beeLine.error("Fail to connect with a local driver due to the exception:" + e); + } catch (ClassNotFoundException e) { + beeLine.error("Fail to connect with a local driver due to the exception:" + e); + } catch (InstantiationException e) { + beeLine.error("Fail to connect with a local driver due to the exception:" + e); + } catch (IllegalAccessException e) { + beeLine.error("Fail to connect with a local driver due to the exception:" + e); + } + } + return null; + } + + public static boolean isSupportedLocalDriver(Driver driver) { + String driverName = driver.getClass().getName(); + for (String name : SUPPORTEDLOCALDRIVERS) { + if (name.equals(driverName)) { + return true; + } + } + return false; + } public Connection getConnection() throws SQLException { if (connection != null) { diff --git beeline/src/main/resources/BeeLine.properties beeline/src/main/resources/BeeLine.properties index d038d46..5fc2e22 100644 --- beeline/src/main/resources/BeeLine.properties +++ beeline/src/main/resources/BeeLine.properties @@ -71,6 +71,7 @@ help-properties: Connect to the database specified in the properties file(s) help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. +help-addlocaljar: Add jar files to the beeline client side. jline-missing: The JLine jar was not found. Please ensure it is installed. diff --git beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java index a6ee93a..aa55bd0 100644 --- beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java +++ beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java @@ -19,17 +19,37 @@ package org.apache.hive.beeline; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; -import junit.framework.Assert; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hive.common.util.HiveTestUtils; +import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * Unit test for Beeline arg parser. */ +@RunWith(Parameterized.class) public class TestBeelineArgParsing { + private static final Log LOG = LogFactory.getLog(TestBeelineArgParsing.class.getName()); + + private String cnnStr; + private String driverClazzName; + private String driverJarFileName; + + public TestBeelineArgParsing(String cnnStr, String driverClazzName, String driverJarFileName) { + this.cnnStr = cnnStr; + this.driverClazzName = driverClazzName; + this.driverJarFileName = driverJarFileName; + } + public class TestBeeline extends BeeLine { String connectArgs = null; @@ -49,6 +69,20 @@ boolean dispatch(String command) { } return true; } + + public boolean addLocalJar(String url){ + String line = "addlocaljar " + url; + return getCommands().addlocaljar(line); + } + } + + @Parameters + public static Collection data() { + return Arrays.asList( + new Object[][]{ + {"jdbc:mysql://host:port/xctest/testdb", "com.mysql.jdbc.Driver", "mysql-connector-java-bin.jar"}, + {"jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver", "postgresql-9.3.jdbc3.jar"} + }); } @Test @@ -147,4 +181,15 @@ public void testUnmatchedArgs() throws Exception { Assert.assertEquals(-1, bl.initArgs(args)); } + @Test + public void testAddLocalJar() throws Exception { + TestBeeline bl = new TestBeeline(); + Assert.assertNull(bl.findLocalDriver(cnnStr)); + + LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName); + String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName); + + bl.addLocalJar(mysqlDriverPath); + Assert.assertEquals(bl.findLocalDriver(cnnStr).getClass().getName(), driverClazzName); + } } diff --git beeline/src/test/resources/mysql-connector-java-bin.jar beeline/src/test/resources/mysql-connector-java-bin.jar new file mode 100644 index 0000000..a10268c Binary files /dev/null and beeline/src/test/resources/mysql-connector-java-bin.jar differ diff --git beeline/src/test/resources/postgresql-9.3.jdbc3.jar beeline/src/test/resources/postgresql-9.3.jdbc3.jar new file mode 100644 index 0000000..f537b98 Binary files /dev/null and beeline/src/test/resources/postgresql-9.3.jdbc3.jar differ