diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 79922d2..164d240 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -241,6 +241,8 @@ new ReflectiveCommandHandler(this, new String[] {"outputformat"}, new Completer[] {new StringsCompleter( formats.keySet().toArray(new String[0]))}), + new ReflectiveCommandHandler(this, new String[] {"disablequoting"}, + new Completer[] {new StringsCompleter (new String[] {"true", "false"})}), new ReflectiveCommandHandler(this, new String[] {"autocommit"}, null), new ReflectiveCommandHandler(this, new String[] {"commit"}, diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 57b9c46..dc371aa 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -88,6 +88,7 @@ int timeout = -1; private String isolation = DEFAULT_ISOLATION_LEVEL; private String outputFormat = "table"; + private boolean disableQuoting = false; // This configuration is used only for client side configuration. private HiveConf conf; private boolean trimScripts = true; @@ -556,6 +557,14 @@ public String getOutputFormat() { return outputFormat; } + public void setDisableQuoting(boolean disableQuoting) { + this.disableQuoting = disableQuoting; + } + + public boolean getDisableQuoting() { + return disableQuoting; + } + public void setTrimScripts(boolean trimScripts) { this.trimScripts = trimScripts; } diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 039e354..fb7639b 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -646,6 +646,9 @@ public boolean outputformat(String line) { return set("set " + line); } + public boolean disablequoting (String line) { + return set("set " + line); + } public boolean brief(String line) { beeLine.info("verbose: off"); diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 66d9fd0..6c0e734 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -33,28 +33,26 @@ * OutputFormat for values separated by a delimiter. */ class SeparatedValuesOutputFormat implements OutputFormat { - public final static String DISABLE_QUOTING_FOR_SV = "disable.quoting.for.sv"; private final BeeLine beeLine; + private char separator; private CsvPreference quotedCsvPreference; - private CsvPreference unquotedCsvPreference; SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; - unquotedCsvPreference = new CsvPreference.Builder('\0', separator, "").build(); + this.separator = separator; quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build(); } private void updateCsvPreference() { if (beeLine.getOpts().getOutputFormat().equals("dsv")) { // check whether delimiter changed by user - char curDel = (char) getCsvPreference().getDelimiterChar(); char newDel = beeLine.getOpts().getDelimiterForDSV(); // if delimiter changed, rebuild the csv preference - if (newDel != curDel) { + if (newDel != separator) { // "" is passed as the end of line symbol in following function, as // beeline itself adds newline if (isQuotingDisabled()) { - unquotedCsvPreference = new CsvPreference.Builder('\0', newDel, "").build(); + separator = newDel; } else { quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build(); } @@ -80,31 +78,39 @@ public int print(Rows rows) { } private String getFormattedStr(String[] vals) { - StringWriter strWriter = new StringWriter(); - CsvListWriter writer = new CsvListWriter(strWriter, getCsvPreference()); - if (vals.length > 0) { - try { - writer.write(vals); - } catch (IOException e) { - beeLine.error(e); - } finally { - IOUtils.closeStream(writer); + if (isQuotingDisabled()) { + String s = ""; + for (int i = 0; i < vals.length; i++) { + s = s + vals[i] + separator; } + return s.substring(0, s.length()-1); + } else { + StringWriter strWriter = new StringWriter(); + CsvListWriter writer = new CsvListWriter(strWriter, quotedCsvPreference); + if (vals.length > 0) { + try { + writer.write(vals); + } catch (IOException e) { + beeLine.error(e); + } finally { + IOUtils.closeStream(writer); + } + } + return strWriter.toString(); } - return strWriter.toString(); } - private void printRow(Rows.Row row) { + public void printRow(Rows.Row row) { String[] vals = row.values; String formattedStr = getFormattedStr(vals); beeLine.output(formattedStr); } private boolean isQuotingDisabled() { - String quotingDisabledStr = System.getProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV); + String quotingDisabledStr = Boolean.toString(beeLine.getOpts().getDisableQuoting()); if (quotingDisabledStr == null || quotingDisabledStr.isEmpty()) { - // default is disabling the double quoting for separated value - return true; + // default is enabling the double quoting for separated value + return false; } String parsedOptionStr = quotingDisabledStr.toLowerCase(); if (parsedOptionStr.equals("false") || parsedOptionStr.equals("true")) { @@ -112,15 +118,7 @@ private boolean isQuotingDisabled() { } else { beeLine.error("System Property disable.quoting.for.sv is now " + parsedOptionStr + " which only accepts boolean value"); - return true; - } - } - - private CsvPreference getCsvPreference() { - if (isQuotingDisabled()) { - return unquotedCsvPreference; - } else { - return quotedCsvPreference; + return false; } } } diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index ad79c01..c0ced3f 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -69,6 +69,7 @@ help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) +help-disablequoting: Disable double quotes around a value in csv2,tsv2 and dsv formats(only if value contains special characters,such as the delimiter or double quote character). By default, the disablequoting is false. help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. @@ -186,6 +187,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --autosave=[true/false] automatically save preferences\n \ \ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv] format mode for result display\n \ \ Note that csv, and tsv are deprecated - use csv2, tsv2 instead\n \ +\ --disablequoting=[true/false] Applied in csv2,tsv2 and dsv formats. The default is set to false. When set to true, double quotes will not be visible around a value(when value contains special characters).\n \ \ --incremental=[true/false] Defaults to false. When set to false, the entire result set\n \ \ is fetched and buffered before being displayed, yielding optimal\n \ \ display column sizing. When set to true, result rows are displayed\n \ diff --git a/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java b/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java new file mode 100644 index 0000000..723da30 --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.beeline; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.PrintStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestOutputFormatWithQuotes { + private BeelineMock mockBeeline; + private ResultSet mockResultSet; + private TestBufferedRows.MockRow mockRow; + private Rows rows; + private final String[] result = new String[7]; + + private final String[][] mockRowData = { + {"key1", "a,aa"}, + {"key2", "b\"bb"}, + {"key3", "c'cc"}, + {"key4", "d|dd"}, + {"key5", "\"eee\""}, + {"key6", "\"\"fff\"\""} + }; + + private final String[] expectedCsv2 = { + "Key,Value", + "key1,a,aa", + "key2,b\"bb", + "key3,c'cc", + "key4,d|dd", + "key5,\"eee\"", + "key6,\"\"fff\"\"" + }; + + private final String[] expectedTsv2 = { + "Key\tValue", + "key1\ta,aa", + "key2\tb\"bb", + "key3\tc'cc", + "key4\td|dd", + "key5\t\"eee\"", + "key6\t\"\"fff\"\"" + }; + + private final String[] expectedDsv = { + "Key|Value", + "key1|a,aa", + "key2|b\"bb", + "key3|c'cc", + "key4|d|dd", + "key5|\"eee\"", + "key6|\"\"fff\"\"" + }; + + private final String[] expectedCsv2WithQuotes = { + "Key,Value", + "key1,\"a,aa\"", + "key2,\"b\"\"bb\"", + "key3,c'cc", + "key4,d|dd", + "key5,\"\"\"eee\"\"\"", + "key6,\"\"\"\"\"fff\"\"\"\"\"" + }; + + private final String[] expectedTsv2WithQuotes = { + "Key\tValue", + "key1\ta,aa", + "key2\t\"b\"\"bb\"", + "key3\tc'cc", + "key4\td|dd", + "key5\t\"\"\"eee\"\"\"", + "key6\t\"\"\"\"\"fff\"\"\"\"\"" + }; + + private final String[] expectedDsvWithQuotes = { + "Key|Value", + "key1|a,aa", + "key2|\"b\"\"bb\"", + "key3|c'cc", + "key4|\"d|dd\"", + "key5|\"\"\"eee\"\"\"", + "key6|\"\"\"\"\"fff\"\"\"\"\"" + }; + + public class BeelineMock extends BeeLine { + + private String lastPrintedLine; + + @Override + final void output(final ColorBuffer msg, boolean newline, PrintStream out) { + lastPrintedLine = msg.toString(); + super.output(msg, newline, out); + } + + private String getLastPrintedLine() { + return lastPrintedLine; + } + } + + @Before + public void beforeTest() throws SQLException { + setupMockData(); + mockBeeline = new BeelineMock(); + rows = new BufferedRows(mockBeeline, mockResultSet); + } + + @Test + public void testCsv2() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(true); + mockBeeline.getOpts().setOutputFormat("csv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, ','); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedCsv2, result); + } + + @Test + public void testTsv2() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(true); + mockBeeline.getOpts().setOutputFormat("tsv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '\t'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedTsv2, result); + } + + @Test + public void testDsv() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(true); + mockBeeline.getOpts().setOutputFormat("dsv"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '|'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedDsv, result); + } + + @Test + public void testCsv2WithQuotes() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(false); + mockBeeline.getOpts().setOutputFormat("csv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, ','); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedCsv2WithQuotes, result); + } + + @Test + public void testTsv2WithQuotes() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(false); + mockBeeline.getOpts().setOutputFormat("tsv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '\t'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedTsv2WithQuotes, result); + } + + @Test + public void testDsvWithQuotes() throws SQLException { + mockBeeline.getOpts().setDisableQuoting(false); + mockBeeline.getOpts().setOutputFormat("dsv"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '|'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedDsvWithQuotes, result); + } + + private String[] printTest(SeparatedValuesOutputFormat outputFormat) throws SQLException { + int count = 0; + while (rows.hasNext()){ + outputFormat.printRow((Rows.Row) rows.next()); + result[count] = mockBeeline.getLastPrintedLine(); + count++; + } + return result; + } + + private String[] printTest(DeprecatedSeparatedValuesOutputFormat outputFormat) throws SQLException { + int count = 0; + while (rows.hasNext()){ + outputFormat.printRow(rows, (Rows.Row) rows.next()); + result[count] = mockBeeline.getLastPrintedLine(); + count++; + } + return result; + } + + private void setupMockData() throws SQLException { + mockResultSet = mock(ResultSet.class); + + ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); + when(mockResultSetMetaData.getColumnCount()).thenReturn(2); + when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("Key"); + when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("Value"); + when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); + + mockRow = new TestBufferedRows.MockRow(); + // returns true as long as there is more data in mockResultData array + when(mockResultSet.next()).thenAnswer(new Answer() { + private int mockRowDataIndex = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + if (mockRowDataIndex < mockRowData.length) { + mockRow.setCurrentRowData(mockRowData[mockRowDataIndex]); + mockRowDataIndex++; + return true; + } else { + return false; + } + } + }); + + when(mockResultSet.getString(Matchers.anyInt())).thenAnswer(new Answer() { + @Override + public String answer(final InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + int index = ((Integer) args[0]); + return mockRow.getColumn(index); + } + }); + } +}