diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index e3c2449..8539a41 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -145,9 +145,11 @@ private final Map formats = map(new Object[] { "vertical", new VerticalOutputFormat(this), "table", new TableOutputFormat(this), - "csv", new SeparatedValuesOutputFormat(this, ','), - "tsv", new SeparatedValuesOutputFormat(this, '\t'), + "csv2", new SeparatedValuesOutputFormat(this, ','), + "tsv2", new SeparatedValuesOutputFormat(this, '\t'), "dsv", new SeparatedValuesOutputFormat(this, BeeLineOpts.DEFAULT_DELIMITER_FOR_DSV), + "csv", new DeprecatedSeparatedValuesOutputFormat(this, ','), + "tsv", new DeprecatedSeparatedValuesOutputFormat(this, '\t'), "xmlattr", new XMLAttributeOutputFormat(this), "xmlelements", new XMLElementOutputFormat(this), }); diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 2a0e9b5..f73fb44 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -427,6 +427,9 @@ public boolean getAutosave() { } public void setOutputFormat(String outputFormat) { + if(outputFormat.equalsIgnoreCase("csv") || outputFormat.equalsIgnoreCase("tsv")) { + beeLine.info("Format " + outputFormat + " is deprecated, please use " + outputFormat + "2"); + } this.outputFormat = outputFormat; } diff --git a/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java new file mode 100644 index 0000000..f49daee --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/DeprecatedSeparatedValuesOutputFormat.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This source file is based on code taken from SQLLine 1.0.2 + * See SQLLine notice in LICENSE + */ +package org.apache.hive.beeline; + + +/** + * OutputFormat for values separated by a delimiter. + * + * Note this does not handle escaping of the quote char. + * The new SeparatedValuesOutputFormat supports that. The formats supported by + * this class are deprecated. + * + */ +class DeprecatedSeparatedValuesOutputFormat implements OutputFormat { + + private final BeeLine beeLine; + private char separator; + + public DeprecatedSeparatedValuesOutputFormat(BeeLine beeLine, char separator) { + this.beeLine = beeLine; + setSeparator(separator); + } + + @Override + public int print(Rows rows) { + int count = 0; + while (rows.hasNext()) { + printRow(rows, (Rows.Row) rows.next()); + count++; + } + return count - 1; // sans header row + } + + public void printRow(Rows rows, Rows.Row row) { + String[] vals = row.values; + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < vals.length; i++) { + buf.append(buf.length() == 0 ? "" : "" + getSeparator()) + .append('\'') + .append(vals[i] == null ? "" : vals[i]) + .append('\''); + } + beeLine.output(buf.toString()); + } + + public void setSeparator(char separator) { + this.separator = separator; + } + + public char getSeparator() { + return this.separator; + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 6bbb046..a2c18c7 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -51,11 +51,14 @@ private void updateCsvPreference() { char newDel = beeLine.getOpts().getDelimiterForDSV(); // if delimiter changed, rebuild the csv preference if (newDel != curDel) { + // "" is passed as the end of line symbol in following function, as + // beeline itself adds newline csvPreference = new CsvPreference.Builder('"', newDel, "").build(); } } } + @Override public int print(Rows rows) { updateCsvPreference(); diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index d40c0dc..d038d46 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -68,7 +68,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv,dsv,tsv,xmlattrs,xmlelements) +help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. @@ -167,7 +167,8 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \ \ --silent=[true/false] be more silent\n \ \ --autosave=[true/false] automatically save preferences\n \ -\ --outputformat=[table/vertical/csv/tsv/dsv] format mode for result display\n \ +\ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv] format mode for result display\n \ +\ Note that csv, and tsv are deprecated - use csv2, tsv2 instead\n\ \ --truncateTable=[true/false] truncate table column when it exceeds length\n \ \ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \ \ --isolation=LEVEL set the transaction isolation level\n \ diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 0947b8d..72462a6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -354,7 +354,8 @@ public void testGetVariableValue() throws Throwable { /** * Select null from table , check if setting null to empty string works. - * Original beeline/sqlline used to print nulls as empty strings + * Original beeline/sqlline used to print nulls as empty strings. + * Also test csv2 output format * Print PASSED or FAILED */ @Test @@ -365,12 +366,105 @@ public void testNullEmpty() throws Throwable { final String EXPECTED_PATTERN = "abc,,def"; List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=csv2"); + + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV (new) format + */ + @Test + public void testDSVOutput() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=dsv"); + argList.add("--delimiterForDSV=;"); + + final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";1.0"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV (new) format + */ + @Test + public void testTSV2Output() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=tsv2"); + + final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t1.0"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV deprecated format + */ + @Test + public void testTSVOutput() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=tsv"); + + final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + + /** + * Test writing output using TSV deprecated format + * Check for deprecation message + */ + @Test + public void testTSVOutputDeprecation() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=tsv"); + + final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using CSV deprecated format + * Check for deprecation message + */ + @Test + public void testCSVOutputDeprecation() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); argList.add("--outputformat=csv"); + final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } /** + * Test writing output using CSV deprecated format + * Check for deprecation message + */ + @Test + public void testCSVOutput() throws Throwable { + final String TEST_NAME = "testTSVOutput"; + String SCRIPT_TEXT = getFormatTestQuery(); + List argList = getBaseArgs(JDBC_URL); + argList.add("--outputformat=csv"); + final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; + testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + + private String getFormatTestQuery() { + return "set hive.support.concurrency = false;\n" + + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; + } + /** * Select null from table , check if setting null to empty string works - Using beeling cmd line * argument. * Original beeline/sqlline used to print nulls as empty strings @@ -381,8 +475,7 @@ public void testNullEmptyCmdArg() throws Throwable { final String TEST_NAME = "testNullNonDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; - //final String EXPECTED_PATTERN = "| abc | | def |"; - final String EXPECTED_PATTERN = "abc,,def"; + final String EXPECTED_PATTERN = "'abc','','def'"; List argList = getBaseArgs(JDBC_URL); argList.add("--nullemptystring=true"); @@ -412,7 +505,7 @@ public void testNegativeScriptFile() throws Throwable { argList.add(scriptFile.getAbsolutePath()); try { - String output = testCommandLineScript(argList, null); + String output = testCommandLineScript(argList, null); long elapsedTime = (System.currentTimeMillis() - startTime)/1000; String time = "(" + elapsedTime + "s)"; if (output.contains(EXPECTED_PATTERN)) {