diff --git a/hbase-handler/src/test/queries/positive/hbase_binary_binary.q b/hbase-handler/src/test/queries/positive/hbase_binary_binary.q new file mode 100644 index 0000000000000000000000000000000000000000..388e5aad4509f97d7724c7ed73f3a1eb4f3f8e71 --- /dev/null +++ b/hbase-handler/src/test/queries/positive/hbase_binary_binary.q @@ -0,0 +1,12 @@ +drop table if exists testhbaseb; +CREATE TABLE testhbaseb (key int, val binary) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( +"hbase.columns.mapping" = ":key,cf:val#b" +); +insert into table testhbaseb values(1, 'hello'); +insert into table testhbaseb values(2, 'hi'); +select * from testhbaseb; +drop table testhbaseb; + + diff --git a/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out b/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out new file mode 100644 index 0000000000000000000000000000000000000000..c828fd19f9692abf15566fdd5ab2375fc5df4f5b --- /dev/null +++ b/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out @@ -0,0 +1,54 @@ +PREHOOK: query: drop table if exists testhbaseb +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists testhbaseb +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testhbaseb (key int, val binary) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( +"hbase.columns.mapping" = ":key,cf:val#b" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testhbaseb +POSTHOOK: query: CREATE TABLE testhbaseb (key int, val binary) +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' +WITH SERDEPROPERTIES ( +"hbase.columns.mapping" = ":key,cf:val#b" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testhbaseb +PREHOOK: query: insert into table testhbaseb values(1, 'hello') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@testhbaseb +POSTHOOK: query: insert into table testhbaseb values(1, 'hello') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@testhbaseb +PREHOOK: query: insert into table testhbaseb values(2, 'hi') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@testhbaseb +POSTHOOK: query: insert into table testhbaseb values(2, 'hi') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@testhbaseb +PREHOOK: query: select * from testhbaseb +PREHOOK: type: QUERY +PREHOOK: Input: default@testhbaseb +#### A masked pattern was here #### +POSTHOOK: query: select * from testhbaseb +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testhbaseb +#### A masked pattern was here #### +1 hello +2 hi +PREHOOK: query: drop table testhbaseb +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testhbaseb +PREHOOK: Output: default@testhbaseb +POSTHOOK: query: drop table testhbaseb +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testhbaseb +POSTHOOK: Output: default@testhbaseb diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java index 010f1f9081c7e80c474a04c7e2acd77aa0470a25..aa19d091b09f27080a199f248d92bc610337f781 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java @@ -29,7 +29,7 @@ private static final Logger LOG = LoggerFactory.getLogger(LazyBinary.class); private static final boolean DEBUG_LOG_ENABLED = LOG.isDebugEnabled(); - LazyBinary(LazyBinaryObjectInspector oi) { + public LazyBinary(LazyBinaryObjectInspector oi) { super(oi); data = new BytesWritable(); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 7aa0fc775ba1dacbafef5b36ffb7c37b20fdfa03..23dbe6a3cea79240ec82bcc4ea908285966e97a3 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyTimestampObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyVoidObjectInspector; +import org.apache.hadoop.hive.serde2.lazydio.LazyDioBinary; import org.apache.hadoop.hive.serde2.lazydio.LazyDioBoolean; import org.apache.hadoop.hive.serde2.lazydio.LazyDioByte; import org.apache.hadoop.hive.serde2.lazydio.LazyDioDouble; @@ -162,6 +163,8 @@ return new LazyDioFloat((LazyFloatObjectInspector) poi); case DOUBLE: return new LazyDioDouble((LazyDoubleObjectInspector) poi); + case BINARY: + return new LazyDioBinary((LazyBinaryObjectInspector) poi); default: throw new RuntimeException("Hive Internal Error: no LazyObject for " + poi); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index 29d6ad870f35533869e24cd8c8076aeb57d84f28..ee391962eb2a2013b132f95a3aef3aec29095d12 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -368,6 +368,12 @@ public static void writePrimitive( dos.writeDouble(d); break; + case BINARY: { + BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o); + out.write(bw.getBytes(), 0, bw.getLength()); + break; + } + default: throw new RuntimeException("Hive internal error."); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java new file mode 100644 index 0000000000000000000000000000000000000000..a3e4cac7a974c62f84903db9bb7946135dc0828e --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazydio; + +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.lazy.LazyBinary; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; + +public class LazyDioBinary extends LazyBinary { + + public LazyDioBinary(LazyBinaryObjectInspector oi) { + super(oi); + } + + LazyDioBinary(LazyDioBinary copy) { + super(copy); + } + + /* (non-Javadoc) + * This provides a LazyBinary like class which can be initialized from data stored in a + * binary format. + * + * @see org.apache.hadoop.hive.serde2.lazy.LazyObject#init + * (org.apache.hadoop.hive.serde2.lazy.ByteArrayRef, int, int) + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + if (bytes == null) { + throw new RuntimeException("bytes cannot be null!"); + } + isNull = false; + byte[] recv = new byte[length]; + System.arraycopy(bytes.getData(), start, recv, 0, length); + data.set(recv, 0, length); + } +}