diff --git beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 649bb63..d646b27 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -92,6 +92,7 @@ private String initFile = null; private String authType = null; private char delimiterForDSV = DEFAULT_DELIMITER_FOR_DSV; + private boolean disableQuotingForSV = true; private Map hiveVariables = new HashMap(); private Map hiveConfVariables = new HashMap(); @@ -511,5 +512,13 @@ public char getDelimiterForDSV() { public void setDelimiterForDSV(char delimiterForDSV) { this.delimiterForDSV = delimiterForDSV; } + + public boolean isDisableQuotingForSV() { + return disableQuotingForSV; + } + + public void setDisableQuotingForSV(boolean disableQuotingForSV) { + this.disableQuotingForSV = disableQuotingForSV; + } } diff --git beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index a2c18c7..ceb9211 100644 --- beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -37,23 +37,30 @@ * */ private final BeeLine beeLine; - private CsvPreference csvPreference; + private CsvPreference quotedCsvPreference; + private CsvPreference unquotedCsvPreference; SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; - csvPreference = new CsvPreference.Builder('"', separator, "").build(); + unquotedCsvPreference = new CsvPreference.Builder('\0', separator, "").build(); + quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build(); } private void updateCsvPreference() { if (beeLine.getOpts().getOutputFormat().equals("dsv")) { // check whether delimiter changed by user - char curDel = (char) csvPreference.getDelimiterChar(); + char curDel = (char) getCsvPreference().getDelimiterChar(); char newDel = beeLine.getOpts().getDelimiterForDSV(); // if delimiter changed, rebuild the csv preference if (newDel != curDel) { // "" is passed as the end of line symbol in following function, as // beeline itself adds newline - csvPreference = new CsvPreference.Builder('"', newDel, "").build(); + boolean isQuotingDisabled = beeLine.getOpts().isDisableQuotingForSV(); + if (isQuotingDisabled) { + unquotedCsvPreference = new CsvPreference.Builder('\0', newDel, "").build(); + } else { + quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build(); + } } } } @@ -61,10 +68,9 @@ private void updateCsvPreference() { @Override public int print(Rows rows) { updateCsvPreference(); - int count = 0; while (rows.hasNext()) { - printRow(rows, (Rows.Row) rows.next()); + printRow((Rows.Row) rows.next()); count++; } return count - 1; // sans header row @@ -72,7 +78,7 @@ public int print(Rows rows) { private String getFormattedStr(String[] vals) { StringWriter strWriter = new StringWriter(); - CsvListWriter writer = new CsvListWriter(strWriter, csvPreference); + CsvListWriter writer = new CsvListWriter(strWriter, getCsvPreference()); if (vals.length > 0) { try { writer.write(vals); @@ -85,9 +91,18 @@ private String getFormattedStr(String[] vals) { return strWriter.toString(); } - public void printRow(Rows rows, Rows.Row row) { + private void printRow(Rows.Row row) { String[] vals = row.values; String formattedStr = getFormattedStr(vals); beeLine.output(formattedStr); } + + private CsvPreference getCsvPreference() { + boolean isQuotingDisabled = beeLine.getOpts().isDisableQuotingForSV(); + if (isQuotingDisabled) { + return unquotedCsvPreference; + } else { + return quotedCsvPreference; + } + } } diff --git beeline/src/main/resources/BeeLine.properties beeline/src/main/resources/BeeLine.properties index 2e987fd..23d1ac4 100644 --- beeline/src/main/resources/BeeLine.properties +++ beeline/src/main/resources/BeeLine.properties @@ -70,6 +70,7 @@ help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format +help-disableQuotingForSV: Disable the quoting for the tsv/csv/dsv output format altogether help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -174,6 +175,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \ \ --isolation=LEVEL set the transaction isolation level\n \ \ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ +\ --disableQuotingForSV=[true/false] set to true to disable quotes for TSV, CSV and DSV output format altogether\n \ \ --help display this message diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 1f3e484..f675323 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -352,7 +352,7 @@ public void testDSVOutput() throws Throwable { argList.add("--outputformat=dsv"); argList.add("--delimiterForDSV=;"); - final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";1.0"; + final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -365,7 +365,7 @@ public void testTSV2Output() throws Throwable { List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); - final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t1.0"; + final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0"; testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } @@ -382,6 +382,76 @@ public void testTSVOutput() throws Throwable { testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); } + /** + * Test writing output using new TSV format + */ + @Test + public void testTSV2OutputWithDoubleQuotes() throws Throwable { + String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv2"); + argList.add("--disableQuotingForSV=false"); + + final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using TSV deprecated format + */ + @Test + public void testTSVOutputWithDoubleQuotes() throws Throwable { + String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=tsv"); + argList.add("--disableQuotingForSV=false"); + + final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using new CSV format + */ + @Test + public void testCSV2OutputWithDoubleQuotes() throws Throwable { + String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=csv2"); + argList.add("--disableQuotingForSV=false"); + + final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using CSV deprecated format + */ + @Test + public void testCSVOutputWithDoubleQuotes() throws Throwable { + String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=csv"); + argList.add("--disableQuotingForSV=false"); + + final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } + + /** + * Test writing output using DSV format, with custom delimiter ";" + */ + @Test + public void testDSVOutputWithDoubleQuotes() throws Throwable { + String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); + List argList = getBaseArgs(miniHS2.getBaseJdbcURL()); + argList.add("--outputformat=dsv"); + argList.add("--delimiterForDSV=;"); + argList.add("--disableQuotingForSV=false"); + + final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0"; + testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + } /** * Test writing output using TSV deprecated format @@ -428,6 +498,12 @@ private String getFormatTestQuery() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; } + + private String getFormatTestQueryForEableQuotes() { + return "set hive.support.concurrency = false;\n" + + "select 1, null, 'defg', 'ab\"c', '\"aa\"', 1.0D from " + tableName + " limit 1 ;\n"; + } + /** * Select null from table , check if setting null to empty string works - Using beeling cmd line * argument.