diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index bc2f4d8..c5e36a5 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -224,6 +224,8 @@ null), new ReflectiveCommandHandler(this, new String[] {"call"}, null), + new ReflectiveCommandHandler(this, new String[] {"nullemptystring"}, + new Completor[] {new BooleanCompletor()}), }; diff --git beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 0884b16..c3abba3 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -52,6 +52,7 @@ public static final String PROPERTY_PREFIX = "beeline."; public static final String PROPERTY_NAME_EXIT = PROPERTY_PREFIX + "system.exit"; + public static final String DEFAULT_NULL_STRING = "NULL"; private final BeeLine beeLine; private boolean autosave = false; @@ -78,6 +79,9 @@ private boolean trimScripts = true; private boolean allowMultiLineCommand = true; + //This can be set for old behavior of nulls printed as empty strings + private boolean nullEmptyString = false; + private final File rcFile = new File(saveDir(), "beeline.properties"); private String historyFile = new File(saveDir(), "history").getAbsolutePath(); @@ -443,5 +447,22 @@ public void setAllowMultiLineCommand(boolean allowMultiLineCommand) { this.allowMultiLineCommand = allowMultiLineCommand; } + /** + * Use getNullString() to get the null string to be used. + * @return true if null representation should be an empty string + */ + public boolean getNullEmptyString() { + return nullEmptyString; + } + + public void setNullEmptyString(boolean nullStringEmpty) { + this.nullEmptyString = nullStringEmpty; + } + + public String getNullString(){ + return nullEmptyString ? "" : DEFAULT_NULL_STRING; + } + + } diff --git beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java new file mode 100644 index 0000000..3e88c53 --- /dev/null +++ beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hive.beeline; + +import java.util.List; + +import jline.Completor; +import jline.SimpleCompletor; + +/** + * JLine completor boolean value (true/false) + */ +class BooleanCompletor extends SimpleCompletor { + + public BooleanCompletor(){ + super(new String[] {"true", "false"}); + } + +} \ No newline at end of file diff --git beeline/src/java/org/apache/hive/beeline/BufferedRows.java beeline/src/java/org/apache/hive/beeline/BufferedRows.java index fc8e62b..962c531 100644 --- beeline/src/java/org/apache/hive/beeline/BufferedRows.java +++ beeline/src/java/org/apache/hive/beeline/BufferedRows.java @@ -54,6 +54,11 @@ public Object next() { } @Override + public String toString(){ + return list.toString(); + } + + @Override void normalizeWidths() { int[] max = null; for (Row row : list) { diff --git beeline/src/java/org/apache/hive/beeline/Rows.java beeline/src/java/org/apache/hive/beeline/Rows.java index a66e709..453f685 100644 --- beeline/src/java/org/apache/hive/beeline/Rows.java +++ beeline/src/java/org/apache/hive/beeline/Rows.java @@ -27,19 +27,23 @@ import java.sql.SQLException; import java.text.DecimalFormat; import java.text.NumberFormat; +import java.util.Arrays; import java.util.Iterator; /** * Abstract base class representing a set of rows to be displayed. + * Holds column values as strings */ abstract class Rows implements Iterator { private final BeeLine beeLine; final ResultSetMetaData rsMeta; final Boolean[] primaryKeys; final NumberFormat numberFormat; + private final String nullStr; Rows(BeeLine beeLine, ResultSet rs) throws SQLException { this.beeLine = beeLine; + nullStr = beeLine.getOpts().getNullString(); rsMeta = rs.getMetaData(); int count = rsMeta.getColumnCount(); primaryKeys = new Boolean[count]; @@ -125,6 +129,10 @@ boolean isPrimaryKey(int col) { inserted = false; } + @Override + public String toString(){ + return Arrays.asList(values).toString(); + } Row(int size, ResultSet rs) throws SQLException { isMeta = false; @@ -157,7 +165,8 @@ boolean isPrimaryKey(int col) { } else { values[i] = rs.getString(i + 1); } - sizes[i] = values[i] == null ? 1 : values[i].length(); + values[i] = values[i] == null ? nullStr : values[i]; + sizes[i] = values[i].length(); } } } diff --git beeline/src/main/resources/BeeLine.properties beeline/src/main/resources/BeeLine.properties index 9947007..408286d 100644 --- beeline/src/main/resources/BeeLine.properties +++ beeline/src/main/resources/BeeLine.properties @@ -67,6 +67,7 @@ help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) help-outputformat: Set the output format for displaying results (table,vertical,csv,tsv,xmlattrs,xmlelements) +help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -163,6 +164,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --autosave=[true/false] automatically save preferences\n \ \ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \ \ --isolation=LEVEL set the transaction isolation level\n \ +\ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ \ --help display this message diff --git beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java index e3c6a72..819a28d 100644 --- beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java +++ beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java @@ -23,9 +23,14 @@ import java.io.FileOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.beeline.BeeLine; import org.apache.hive.service.server.HiveServer2; @@ -34,6 +39,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import static org.junit.Assert.*; + /** * TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine * @@ -42,6 +49,9 @@ public class TestBeeLineWithArgs { // Default location of HiveServer2 final private static String JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL + "localhost:10000"; + private static final String tableName = "TestBeelineTable1"; + private static final String tableComment = "Test table comment"; + private static HiveServer2 hiveServer2; @@ -67,6 +77,45 @@ public static void preTests() throws Exception { System.err.println("Starting HiveServer2..."); hiveServer2.start(); Thread.sleep(1000); + createTable(); + + } + + /** + * Create table for use by tests + * @throws ClassNotFoundException + * @throws SQLException + */ + private static void createTable() throws ClassNotFoundException, SQLException { + Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); + Connection con = DriverManager.getConnection(JDBC_URL,"", ""); + + assertNotNull("Connection is null", con); + assertFalse("Connection should not be closed", con.isClosed()); + Statement stmt = con.createStatement(); + assertNotNull("Statement is null", stmt); + + stmt.execute("set hive.support.concurrency = false"); + + HiveConf conf = new HiveConf(); + String dataFileDir = conf.get("test.data.files").replace('\\', '/') + .replace("c:", ""); + Path dataFilePath = new Path(dataFileDir, "kv1.txt"); + // drop table. ignore error. + try { + stmt.execute("drop table " + tableName); + } catch (Exception ex) { + fail(ex.toString()); + } + + // create table + stmt.execute("create table " + tableName + + " (under_col int comment 'the under column', value string) comment '" + + tableComment + "'"); + + // load data + stmt.execute("load data local inpath '" + + dataFilePath.toString() + "' into table " + tableName); } /** @@ -129,39 +178,12 @@ private void testScriptFile(String testName, String scriptText, String expectedP argList.add("-f"); argList.add(scriptFile.getAbsolutePath()); - if(shouldMatch) { - try { - String output = testCommandLineScript(argList); - long elapsedTime = (System.currentTimeMillis() - startTime)/1000; - String time = "(" + elapsedTime + "s)"; - if (output.contains(expectedPattern)) { - System.out.println(">>> PASSED " + testName + " " + time); - } else { - System.err.println("Output: " + output); - System.err.println(">>> FAILED " + testName + " (ERROR) " + time); - Assert.fail(testName); - } - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - } else { - try { - String output = testCommandLineScript(argList); - long elapsedTime = (System.currentTimeMillis() - startTime)/1000; - String time = "(" + elapsedTime + "s)"; - if (output.contains(expectedPattern)) { - System.err.println("Output: " + output); - System.err.println(">>> FAILED " + testName + " (ERROR) " + time); - Assert.fail(testName); - } else { - System.out.println(">>> PASSED " + testName + " " + time); - } - } catch (Throwable e) { - System.err.println("Exception: " + e.toString()); - e.printStackTrace(); - throw e; - } + String output = testCommandLineScript(argList); + boolean matches = output.contains(expectedPattern); + if (shouldMatch != matches) { + //failed + fail(testName + ": Output" + output + " should" + (shouldMatch ? "" : " not") + + " contain " + expectedPattern); } scriptFile.delete(); } @@ -212,6 +234,72 @@ public void testBreakOnErrorScriptFile() throws Throwable { testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); } + + /** + * Select null from table , check how null is printed + * Print PASSED or FAILED + */ + @Test + public void testNullDefault() throws Throwable { + final String TEST_NAME = "testNullDefault"; + final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + + "select null from " + tableName + " limit 1 ;\n"; + final String EXPECTED_PATTERN = "NULL"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + } + + /** + * Select null from table , check if default null is printed differently + * Print PASSED or FAILED + */ + @Test + public void testNullNonEmpty() throws Throwable { + final String TEST_NAME = "testNullNonDefault"; + final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; + final String EXPECTED_PATTERN = "NULL"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL)); + } + + /** + * Select null from table , check if setting null to empty string works. + * Original beeline/sqlline used to print nulls as empty strings + * Print PASSED or FAILED + */ + @Test + public void testNullEmpty() throws Throwable { + final String TEST_NAME = "testNullNonDefault"; + final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + final String EXPECTED_PATTERN = "'abc','','def'"; + + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=csv"); + + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Select null from table , check if setting null to empty string works - Using beeling cmd line + * argument. + * Original beeline/sqlline used to print nulls as empty strings + * Print PASSED or FAILED + */ + @Test + public void testNullEmptyCmdArg() throws Throwable { + final String TEST_NAME = "testNullNonDefault"; + final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + //final String EXPECTED_PATTERN = "| abc | | def |"; + final String EXPECTED_PATTERN = "'abc','','def'"; + + List argList = getBaseArgs(JDBC_URL); + argList.add("--nullemptystring=true"); + argList.add("--outputformat=csv"); + + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + /** * Attempt to execute a missing script file with the -f option to BeeLine * Print PASSED or FAILED @@ -239,7 +327,7 @@ public void testNegativeScriptFile() throws Throwable { if (output.contains(EXPECTED_PATTERN)) { System.err.println("Output: " + output); System.err.println(">>> FAILED " + TEST_NAME + " (ERROR) " + time); - Assert.fail(TEST_NAME); + fail(TEST_NAME); } else { System.out.println(">>> PASSED " + TEST_NAME + " " + time); }