diff --git hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index 56e7a4a..25ae06b 100644 --- hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -189,7 +189,7 @@ public class HBaseSerDe implements SerDe { if (parts.length == 2) { columnMapping.qualifierName = parts[1]; - columnMapping.qualifierNameBytes = Bytes.toBytes(parts[1]); + columnMapping.qualifierNameBytes = Bytes.toBytesBinary(parts[1]); } else { columnMapping.qualifierName = null; columnMapping.qualifierNameBytes = null; @@ -546,10 +546,11 @@ public class HBaseSerDe implements SerDe { throw new SerDeException("HBase row key cannot be NULL"); } - if(putTimestamp >= 0) + if(putTimestamp >= 0) { put = new Put(key,putTimestamp); - else + } else { put = new Put(key); + } // Serialize each field for (int i = 0; i < fields.size(); i++) { diff --git hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java index cbeecd3..185f225 100644 --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.hbase; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -688,4 +689,80 @@ public class TestHBaseSerDe extends TestCase { Put serializedPut = (Put) hbaseSerDe.serialize(row, soi); assertEquals("Serialized data: ", p.toString(), serializedPut.toString()); } + + public void testHBaseSerDeBinaryColumnName() throws IOException, SerDeException { + byte[] cfa = "cola".getBytes(); + byte[] cfb = "colb".getBytes(); + byte[] cfc = "colc".getBytes(); + + byte[] qualByte = Bytes.vintToBytes(1); + byte[] qualShort = Bytes.vintToBytes(2); + byte[] qualInt = Bytes.vintToBytes(3); + byte[] qualLong = Bytes.vintToBytes(4); + byte[] qualFloat = Bytes.vintToBytes(5); + byte[] qualDouble = Bytes.vintToBytes(6); + byte[] qualString = Bytes.vintToBytes(7); + byte[] qualBool = Bytes.vintToBytes(8); + + byte[] rowKey = Bytes.toBytes("test-row1"); + + // Data + List kvs = new ArrayList(); + + kvs.add(new KeyValue(rowKey, cfa, qualByte, Bytes.toBytes("123"))); + kvs.add(new KeyValue(rowKey, cfb, qualShort, Bytes.toBytes("456"))); + kvs.add(new KeyValue(rowKey, cfc, qualInt, Bytes.toBytes("789"))); + kvs.add(new KeyValue(rowKey, cfa, qualLong, Bytes.toBytes("1000"))); + kvs.add(new KeyValue(rowKey, cfb, qualFloat, Bytes.toBytes("-0.01"))); + kvs.add(new KeyValue(rowKey, cfc, qualDouble, Bytes.toBytes("5.3"))); + kvs.add(new KeyValue(rowKey, cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive"))); + kvs.add(new KeyValue(rowKey, cfb, qualBool, Bytes.toBytes("true"))); + Collections.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + + Put p = new Put(rowKey); + + p.add(cfa, qualByte, Bytes.toBytes("123")); + p.add(cfb, qualShort, Bytes.toBytes("456")); + p.add(cfc, qualInt, Bytes.toBytes("789")); + p.add(cfa, qualLong, Bytes.toBytes("1000")); + p.add(cfb, qualFloat, Bytes.toBytes("-0.01")); + p.add(cfc, qualDouble, Bytes.toBytes("5.3")); + p.add(cfa, qualString, Bytes.toBytes("Hadoop, HBase, and Hive")); + p.add(cfb, qualBool, Bytes.toBytes("true")); + + Object[] expectedFieldsData = { + new Text("test-row1"), + new ByteWritable((byte) 123), + new ShortWritable((short) 456), + new IntWritable(789), + new LongWritable(1000), + new FloatWritable(-0.01F), + new DoubleWritable(5.3), + new Text("Hadoop, HBase, and Hive"), + new BooleanWritable(true) + }; + + // Create, initialize, and test the SerDe + HBaseSerDe serDe = new HBaseSerDe(); + Configuration conf = new Configuration(); + Properties tbl = createPropertiesForHiveBinaryColumnName(); + serDe.initialize(conf, tbl); + + deserializeAndSerialize(serDe, r, p, expectedFieldsData); + } + + private Properties createPropertiesForHiveBinaryColumnName() { + Properties tbl = new Properties(); + + // Set the configuration parameters + tbl.setProperty(Constants.LIST_COLUMNS, + "key,abyte,ashort,aint,along,afloat,adouble,astring,abool"); + tbl.setProperty(Constants.LIST_COLUMN_TYPES, + "string,tinyint:smallint:int:bigint:float:double:string:boolean"); + tbl.setProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING, + "cola:\\x01,colb:\\x02,colc:\\x03,cola:\\x04,colb:\\x05,colc:\\x06,cola:\\x07,colb:\\x08"); + return tbl; + } }