diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java index b57e4ab518..114c205c83 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java @@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.CharacterCodingException; import java.sql.Date; import java.sql.Timestamp; import java.util.ArrayList; @@ -50,6 +51,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; @@ -69,6 +71,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hive.common.util.HiveStringUtils; @@ -301,8 +304,20 @@ private Object extractCurrentField(JsonParser p, HCatFieldSchema hcatFieldSchema val = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText(); break; case BINARY: - throw new IOException("JsonSerDe does not support BINARY type"); - case DATE: + String b = (valueToken == JsonToken.VALUE_NULL) ? null : p.getText(); + if (b != null) { + try { + String t = Text.decode(b.getBytes(), 0, b.getBytes().length); + return t.getBytes(); + } catch (CharacterCodingException e) { + LOG.warn("Error generating json binary type from object.", e); + return null; + } + } else { + val = null; + } + break; + case DATE: val = (valueToken == JsonToken.VALUE_NULL) ? null : Date.valueOf(p.getText()); break; case TIMESTAMP: @@ -394,7 +409,13 @@ private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveTypeInfo case STRING: return s; case BINARY: - throw new IOException("JsonSerDe does not support BINARY type"); + try { + String t = Text.decode(s.getBytes(), 0, s.getBytes().length); + return t.getBytes(); + } catch (CharacterCodingException e) { + LOG.warn("Error generating json binary type from object.", e); + return null; + } case DATE: return Date.valueOf(s); case TIMESTAMP: @@ -497,9 +518,12 @@ private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector appendWithQuotes(sb, s); break; } - case BINARY: { - throw new IOException("JsonSerDe does not support BINARY type"); - } + case BINARY: + byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o); + Text txt = new Text(); + txt.set(b, 0, b.length); + appendWithQuotes(sb, SerDeUtils.escapeString(txt.toString())); + break; case DATE: Date d = ((DateObjectInspector)poi).getPrimitiveJavaObject(o); appendWithQuotes(sb, d.toString()); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java index c5753c5be4..8aeb4f4f10 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java @@ -18,6 +18,7 @@ */ package org.apache.hive.hcatalog.data; +import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; @@ -46,7 +47,7 @@ private static final Logger LOG = LoggerFactory.getLogger(TestJsonSerDe.class); - public List> getData() { + public List> getData() throws UnsupportedEncodingException { List> data = new ArrayList>(); List rlist = new ArrayList(13); @@ -98,6 +99,7 @@ rlist.add(new HiveVarchar("hive\nvarchar", 20)); rlist.add(Date.valueOf("2014-01-07")); rlist.add(new Timestamp(System.currentTimeMillis())); + rlist.add("hive\nbinary".getBytes("UTF-8")); List nlist = new ArrayList(13); nlist.add(null); // tinyint @@ -118,15 +120,16 @@ nlist.add(null); //varchar(20) nlist.add(null); //date nlist.add(null); //timestamp + nlist.add(null); //binary String typeString = "tinyint,smallint,int,bigint,double,float,string,string," + "struct,array,map,boolean," + "array,ii2:map>>>>," + - "decimal(5,2),char(10),varchar(20),date,timestamp"; + "decimal(5,2),char(10),varchar(20),date,timestamp,binary"; Properties props = new Properties(); - props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts"); + props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin"); props.put(serdeConstants.LIST_COLUMN_TYPES, typeString); // props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N"); // props.put(Constants.SERIALIZATION_FORMAT, "1");