diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 79922d2..2f24d37 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -241,6 +241,8 @@ new ReflectiveCommandHandler(this, new String[] {"outputformat"}, new Completer[] {new StringsCompleter( formats.keySet().toArray(new String[0]))}), + new ReflectiveCommandHandler(this, new String[] {"enablequoting"}, + new Completer[] {new StringsCompleter (new String[] {"true", "false"})}), new ReflectiveCommandHandler(this, new String[] {"autocommit"}, null), new ReflectiveCommandHandler(this, new String[] {"commit"}, diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 57b9c46..2a54df9 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -88,6 +88,7 @@ int timeout = -1; private String isolation = DEFAULT_ISOLATION_LEVEL; private String outputFormat = "table"; + private boolean enableQuoting = true; // This configuration is used only for client side configuration. private HiveConf conf; private boolean trimScripts = true; @@ -556,6 +557,14 @@ public String getOutputFormat() { return outputFormat; } + public void setEnableQuoting(boolean enableQuoting) { + this.enableQuoting = enableQuoting; + } + + public boolean getEnableQuoting() { + return enableQuoting; + } + public void setTrimScripts(boolean trimScripts) { this.trimScripts = trimScripts; } diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 039e354..0627767 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -646,6 +646,9 @@ public boolean outputformat(String line) { return set("set " + line); } + public boolean enablequoting (String line) { + return set ("set " + line); + } public boolean brief(String line) { beeLine.info("verbose: off"); diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 66d9fd0..32429e5 100644 --- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -33,14 +33,13 @@ * OutputFormat for values separated by a delimiter. */ class SeparatedValuesOutputFormat implements OutputFormat { - public final static String DISABLE_QUOTING_FOR_SV = "disable.quoting.for.sv"; private final BeeLine beeLine; private CsvPreference quotedCsvPreference; private CsvPreference unquotedCsvPreference; SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; - unquotedCsvPreference = new CsvPreference.Builder('\0', separator, "").build(); + unquotedCsvPreference = new CsvPreference.Builder('\u0020', separator, "").surroundingSpacesNeedQuotes(true).build(); quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build(); } @@ -53,8 +52,8 @@ private void updateCsvPreference() { if (newDel != curDel) { // "" is passed as the end of line symbol in following function, as // beeline itself adds newline - if (isQuotingDisabled()) { - unquotedCsvPreference = new CsvPreference.Builder('\0', newDel, "").build(); + if (isQuotingEnabled()) { + unquotedCsvPreference = new CsvPreference.Builder('\u0020', newDel, "").surroundingSpacesNeedQuotes(true).build(); } else { quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build(); } @@ -94,33 +93,33 @@ private String getFormattedStr(String[] vals) { return strWriter.toString(); } - private void printRow(Rows.Row row) { + public void printRow(Rows.Row row) { String[] vals = row.values; String formattedStr = getFormattedStr(vals); beeLine.output(formattedStr); } - private boolean isQuotingDisabled() { - String quotingDisabledStr = System.getProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV); - if (quotingDisabledStr == null || quotingDisabledStr.isEmpty()) { - // default is disabling the double quoting for separated value - return true; + private boolean isQuotingEnabled() { + String quotingEnabledStr = Boolean.toString(beeLine.getOpts().getEnableQuoting()); + if (quotingEnabledStr == null || quotingEnabledStr.isEmpty()) { + // default is enabling the double quoting for separated value + return false; } - String parsedOptionStr = quotingDisabledStr.toLowerCase(); + String parsedOptionStr = quotingEnabledStr.toLowerCase(); if (parsedOptionStr.equals("false") || parsedOptionStr.equals("true")) { return Boolean.parseBoolean(parsedOptionStr); } else { beeLine.error("System Property disable.quoting.for.sv is now " + parsedOptionStr + " which only accepts boolean value"); - return true; + return false; } } private CsvPreference getCsvPreference() { - if (isQuotingDisabled()) { - return unquotedCsvPreference; - } else { + if (isQuotingEnabled()) { return quotedCsvPreference; + } else { + return unquotedCsvPreference; } } } diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index ad79c01..96cdb69 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -69,6 +69,7 @@ help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) +help-enablequoting: Enable double quotes around a value if it contains special characters (such as the delimiter or double quote character). By default, the enablequoting is true. help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. help-addlocaldriverjar: Add driver jar file in the beeline client side. @@ -186,6 +187,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --autosave=[true/false] automatically save preferences\n \ \ --outputformat=[table/vertical/csv2/tsv2/dsv/csv/tsv] format mode for result display\n \ \ Note that csv, and tsv are deprecated - use csv2, tsv2 instead\n \ +\ --enablequoting=[true/false] Defaults to true. When set to false, double quotes will not be visible around a value.\n \ \ --incremental=[true/false] Defaults to false. When set to false, the entire result set\n \ \ is fetched and buffered before being displayed, yielding optimal\n \ \ display column sizing. When set to true, result rows are displayed\n \ diff --git a/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java b/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java new file mode 100644 index 0000000..597c57c --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestOutputFormatWithQuotes.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.beeline; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.PrintStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +import static org.junit.Assert.assertArrayEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestOutputFormatWithQuotes { + private BeelineMock mockBeeline; + private ResultSet mockResultSet; + private TestBufferedRows.MockRow mockRow; + private Rows rows; + private final String[] result = new String[6]; + + private final String[][] mockRowData = { + {"key1", "a,aa"}, + {"key2", "b\"bb"}, + {"key3", "c'cc"}, + {"key4", "d|dd"}, + {"key5", "\"fff\""} + }; + + private final String[] expectedCsv2 = { + "Key,Value", + "key1,\"a,aa\"", + "key2,\"b\"\"bb\"", + "key3,c'cc", + "key4,d|dd", + "key5,\"\"\"fff\"\"\"" + }; + + private final String[] expectedTsv2 = { + "Key\tValue", + "key1\ta,aa", + "key2\t\"b\"\"bb\"", + "key3\tc'cc", + "key4\td|dd", + "key5\t\"\"\"fff\"\"\"" + }; + + private final String[] expectedDsv = { + "Key|Value", + "key1|a,aa", + "key2|\"b\"\"bb\"", + "key3|c'cc", + "key4|\"d|dd\"", + "key5|\"\"\"fff\"\"\"" + }; + + private final String[] expectedCsv = { + "'Key','Value'", + "'key1','a,aa'", + "'key2','b\"bb'", + "'key3','c'cc'", + "'key4','d|dd'", + "'key5','\"fff\"'" + }; + + private final String[] expectedTsv = { + "'Key'\t'Value'", + "'key1'\t'a,aa'", + "'key2'\t'b\"bb'", + "'key3'\t'c'cc'", + "'key4'\t'd|dd'", + "'key5'\t'\"fff\"'" + }; + + public class BeelineMock extends BeeLine { + + private String lastPrintedLine; + + @Override + final void output(final ColorBuffer msg, boolean newline, PrintStream out) { + lastPrintedLine = msg.toString(); + super.output(msg, newline, out); + } + + private String getLastPrintedLine() { + return lastPrintedLine; + } + } + + @Before + public void beforeTest() throws SQLException { + setupMockData(); + mockBeeline = new BeelineMock(); + mockBeeline.getOpts().setEnableQuoting(true); + rows = new BufferedRows(mockBeeline, mockResultSet); + } + + @Test + public void testCsv2 () throws SQLException { + mockBeeline.getOpts().setOutputFormat("csv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, ','); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedCsv2, result); + } + + @Test + public void testTsv2 () throws SQLException { + mockBeeline.getOpts().setOutputFormat("tsv2"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '\t'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedTsv2, result); + } + + @Test + public void testDsv() throws SQLException { + beforeTest(); + mockBeeline.getOpts().setOutputFormat("dsv"); + SeparatedValuesOutputFormat outputFormat = new SeparatedValuesOutputFormat(mockBeeline, '|'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedDsv, result); + } + + @Test + public void testCsv() throws SQLException { + beforeTest(); + mockBeeline.getOpts().setOutputFormat("csv"); + DeprecatedSeparatedValuesOutputFormat outputFormat = new DeprecatedSeparatedValuesOutputFormat(mockBeeline, ','); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedCsv, result); + } + + @Test + public void testTsv() throws SQLException { + beforeTest(); + mockBeeline.getOpts().setOutputFormat("tsv"); + DeprecatedSeparatedValuesOutputFormat outputFormat = new DeprecatedSeparatedValuesOutputFormat(mockBeeline, '\t'); + String[] result = printTest(outputFormat); + assertArrayEquals(expectedTsv, result); + } + + private String[] printTest(SeparatedValuesOutputFormat outputFormat) throws SQLException { + int count = 0; + while (rows.hasNext()){ + outputFormat.printRow((Rows.Row) rows.next()); + result[count] = mockBeeline.getLastPrintedLine(); + count++; + } + return result; + } + + private String[] printTest(DeprecatedSeparatedValuesOutputFormat outputFormat) throws SQLException { + int count = 0; + while (rows.hasNext()){ + outputFormat.printRow(rows, (Rows.Row) rows.next()); + result[count] = mockBeeline.getLastPrintedLine(); + count++; + } + return result; + } + + private void setupMockData() throws SQLException { + mockResultSet = mock(ResultSet.class); + + ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); + when(mockResultSetMetaData.getColumnCount()).thenReturn(2); + when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("Key"); + when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("Value"); + when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); + + mockRow = new TestBufferedRows.MockRow(); + // returns true as long as there is more data in mockResultData array + when(mockResultSet.next()).thenAnswer(new Answer() { + private int mockRowDataIndex = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + if (mockRowDataIndex < mockRowData.length) { + mockRow.setCurrentRowData(mockRowData[mockRowDataIndex]); + mockRowDataIndex++; + return true; + } else { + return false; + } + } + }); + + when(mockResultSet.getString(Matchers.anyInt())).thenAnswer(new Answer() { + @Override + public String answer(final InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + int index = ((Integer) args[0]); + return mockRow.getColumn(index); + } + }); + } + +}