diff --git beeline/pom.xml beeline/pom.xml index 6ec1d1a..f16ca32 100644 --- beeline/pom.xml +++ beeline/pom.xml @@ -81,6 +81,11 @@ libthrift ${libthrift.version} + + net.sf.supercsv + super-csv + ${super-csv.version} + org.apache.hive diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index 528a98e..521f3a2 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -79,13 +79,13 @@ import jline.FileNameCompletor; import jline.History; import jline.SimpleCompletor; -import org.apache.hadoop.io.IOUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.io.IOUtils; /** @@ -148,6 +148,7 @@ "table", new TableOutputFormat(this), "csv", new SeparatedValuesOutputFormat(this, ','), "tsv", new SeparatedValuesOutputFormat(this, '\t'), + "dsv", new SeparatedValuesOutputFormat(this, BeeLineOpts.DEFAULT_DELIMITER_FOR_DSV), "xmlattr", new XMLAttributeOutputFormat(this), "xmlelements", new XMLElementOutputFormat(this), }); diff --git beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java index 75f7d38..403cf5d 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java +++ beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java @@ -53,6 +53,7 @@ public static final String PROPERTY_NAME_EXIT = PROPERTY_PREFIX + "system.exit"; public static final String DEFAULT_NULL_STRING = "NULL"; + public static final char DEFAULT_DELIMITER_FOR_DSV = '|'; private final BeeLine beeLine; private boolean autosave = false; @@ -88,7 +89,7 @@ private String scriptFile = null; private String initFile = null; private String authType = null; - + private char delimiterForDSV = DEFAULT_DELIMITER_FOR_DSV; private Map hiveVariables = new HashMap(); private Map hiveConfVariables = new HashMap(); @@ -490,5 +491,13 @@ public String getNullString(){ public void setHiveConfVariables(Map hiveConfVariables) { this.hiveConfVariables = hiveConfVariables; } + + public char getDelimiterForDSV() { + return delimiterForDSV; + } + + public void setDelimiterForDSV(char delimiterForDSV) { + this.delimiterForDSV = delimiterForDSV; + } } diff --git beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java index 7853c3f..6bbb046 100644 --- beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java +++ beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java @@ -22,25 +22,43 @@ */ package org.apache.hive.beeline; +import java.io.IOException; +import java.io.StringWriter; + +import org.apache.hadoop.io.IOUtils; +import org.supercsv.io.CsvListWriter; +import org.supercsv.prefs.CsvPreference; + /** * OutputFormat for values separated by a delimiter. - * - * TODO: Handle character escaping - * */ class SeparatedValuesOutputFormat implements OutputFormat { /** * */ private final BeeLine beeLine; - private char separator; + private CsvPreference csvPreference; - public SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { + SeparatedValuesOutputFormat(BeeLine beeLine, char separator) { this.beeLine = beeLine; - setSeparator(separator); + csvPreference = new CsvPreference.Builder('"', separator, "").build(); + } + + private void updateCsvPreference() { + if (beeLine.getOpts().getOutputFormat().equals("dsv")) { + // check whether delimiter changed by user + char curDel = (char) csvPreference.getDelimiterChar(); + char newDel = beeLine.getOpts().getDelimiterForDSV(); + // if delimiter changed, rebuild the csv preference + if (newDel != curDel) { + csvPreference = new CsvPreference.Builder('"', newDel, "").build(); + } + } } public int print(Rows rows) { + updateCsvPreference(); + int count = 0; while (rows.hasNext()) { printRow(rows, (Rows.Row) rows.next()); @@ -49,23 +67,24 @@ public int print(Rows rows) { return count - 1; // sans header row } - public void printRow(Rows rows, Rows.Row row) { - String[] vals = row.values; - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < vals.length; i++) { - buf.append(buf.length() == 0 ? "" : "" + getSeparator()) - .append('\'') - .append(vals[i] == null ? "" : vals[i]) - .append('\''); + private String getFormattedStr(String[] vals) { + StringWriter strWriter = new StringWriter(); + CsvListWriter writer = new CsvListWriter(strWriter, csvPreference); + if (vals.length > 0) { + try { + writer.write(vals); + } catch (IOException e) { + beeLine.error(e); + } finally { + IOUtils.closeStream(writer); + } } - beeLine.output(buf.toString()); + return strWriter.toString(); } - public void setSeparator(char separator) { - this.separator = separator; - } - - public char getSeparator() { - return this.separator; + public void printRow(Rows rows, Rows.Row row) { + String[] vals = row.values; + String formattedStr = getFormattedStr(vals); + beeLine.output(formattedStr); } } diff --git beeline/src/main/resources/BeeLine.properties beeline/src/main/resources/BeeLine.properties index 390d062..ba65efc 100644 --- beeline/src/main/resources/BeeLine.properties +++ beeline/src/main/resources/BeeLine.properties @@ -68,7 +68,7 @@ help-procedures: List all the procedures help-tables: List all the tables in the database help-columns: List all the columns for the specified table help-properties: Connect to the database specified in the properties file(s) -help-outputformat: Set the output format for displaying results (table,vertical,csv,tsv,xmlattrs,xmlelements) +help-outputformat: Set the output format for displaying results (table,vertical,csv,dsv,tsv,xmlattrs,xmlelements) help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -166,7 +166,8 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \ \ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \ \ --silent=[true/false] be more silent\n \ \ --autosave=[true/false] automatically save preferences\n \ -\ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \ +\ --outputformat=[table/vertical/csv/tsv/dsv] format mode for result display\n \ +\ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \ \ --isolation=LEVEL set the transaction isolation level\n \ \ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ \ --help display this message diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 8888bd9..e1d44ec 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -362,7 +362,7 @@ public void testNullEmpty() throws Throwable { final String TEST_NAME = "testNullNonDefault"; final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; - final String EXPECTED_PATTERN = "'abc','','def'"; + final String EXPECTED_PATTERN = "abc,,def"; List argList = getBaseArgs(JDBC_URL); argList.add("--outputformat=csv"); @@ -382,7 +382,7 @@ public void testNullEmptyCmdArg() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; //final String EXPECTED_PATTERN = "| abc | | def |"; - final String EXPECTED_PATTERN = "'abc','','def'"; + final String EXPECTED_PATTERN = "abc,,def"; List argList = getBaseArgs(JDBC_URL); argList.add("--nullemptystring=true"); diff --git pom.xml pom.xml index b5a5697..57c1c12 100644 --- pom.xml +++ pom.xml @@ -144,6 +144,7 @@ 1.0.1 1.7.5 4.0.4 + 2.2.0 0.4.0-incubating 1.1 0.2