diff --git a/data/files/non_ascii_tbl.txt b/data/files/non_ascii_tbl.txt new file mode 100644 index 0000000..41586d6 --- /dev/null +++ b/data/files/non_ascii_tbl.txt @@ -0,0 +1 @@ +1|Garçu Kôkaku kidôtai diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 0163788..218a39d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -82,6 +82,7 @@ private static final String dataTypeTableName = "testdatatypetable"; private static final String dataTypeTableComment = "Table with many column data types"; private final HiveConf conf; + public static String dataFileDir; private final Path dataFilePath; private final Path dataTypeDataFilePath; private Connection con; @@ -90,7 +91,7 @@ public TestJdbcDriver2() { conf = new HiveConf(TestJdbcDriver2.class); - String dataFileDir = conf.get("test.data.files").replace('\\', '/') + dataFileDir = conf.get("test.data.files").replace('\\', '/') .replace("c:", ""); dataFilePath = new Path(dataFileDir, "kv1.txt"); dataTypeDataFilePath = new Path(dataFileDir, "datatypes.txt"); @@ -2071,4 +2072,40 @@ public Integer evaluate(final Integer value) { return value; } } + + /** + * Loads data from a table containing non-ascii value column + * Runs a query and compares the return value + * @throws Exception + */ + @Test + public void testNonAsciiReturnValues() throws Exception { + String nonAsciiTableName = "nonAsciiTable"; + String nonAsciiString = "Garçu Kôkaku kidôtai"; + Path nonAsciiFilePath = new Path(dataFileDir, "non_ascii_tbl.txt"); + Statement stmt = con.createStatement(); + stmt.execute("set hive.support.concurrency = false"); + + // Create table + stmt.execute("create table " + nonAsciiTableName + " (key int, value string) " + + "row format delimited fields terminated by '|'"); + + // Load data + stmt.execute("load data local inpath '" + + nonAsciiFilePath.toString() + "' into table " + nonAsciiTableName); + + ResultSet rs = stmt.executeQuery("select value from " + nonAsciiTableName + " limit 1"); + while(rs.next()) { + String resultValue = rs.getString(1); + assertTrue(resultValue.equalsIgnoreCase(nonAsciiString)); + } + + // Drop table, ignore error. + try { + stmt.execute("drop table " + nonAsciiTableName); + } catch (Exception ex) { + // no-op + } + stmt.close(); + } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 6c604ce..46946ab 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.Serializable; +import java.io.UnsupportedEncodingException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -291,7 +292,11 @@ private RowSet decodeFromString(List rows, RowSet rowSet) int protocol = getProtocolVersion().getValue(); for (Object rowString : rows) { - rowObj = serde.deserialize(new BytesWritable(((String)rowString).getBytes())); + try { + rowObj = serde.deserialize(new BytesWritable(((String)rowString).getBytes("UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new SerDeException(e); + } for (int i = 0; i < fieldRefs.size(); i++) { StructField fieldRef = fieldRefs.get(i); fieldOI = fieldRef.getFieldObjectInspector();