[IMPALA-2121] Support implicit columns like timestamp in HBase tables

• Type: Improvement
• Status: Open
• Priority: Minor
• Resolution: Unresolved
• Affects Version/s: Impala 2.2
• Component/s: Frontend
• Target Version: Product Backlog
• Assignee: Unassigned
• Reporter: Gautam Gopalakrishnan
• Created: 03/Jul/15 01:05
• Updated: 20/Jun/17 00:50

Description

With HIVE-2828, support was added for the implicit timestamp column in HBase. So we can define tables like this in Hive

hive> create external table t2 (key string, value string, time timestamp) STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf1:c1, :timestamp") TBLPROPERTIES("hbase.table.name" = "t1");
hive> select * from t2;
OK
r1	val1	2015-07-02 01:34:59.956
Time taken: 2.631 seconds, Fetched: 1 row(s)

This is not usable in the Impala Shell

> select * from t2;
Query: select * from t2
WARNINGS: NoSuchColumnFamilyException: org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family  does not exist in region t1,,1435800893075.ef95505bd60b47c29782024f82fbc19d. in table 't1', {NAME => 'cf1', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '1', TTL => 'FOREVER', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}
	at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5546)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1935)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3176)
	at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30063)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
	at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
	at java.lang.Thread.run(Thread.java:745)

CAUSED BY: RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family  does not exist in region t1,,1435800893075.ef95505bd60b47c29782024f82fbc19d. in table 't1', {NAME => 'cf1', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '1', TTL => 'FOREVER', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}
	at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5546)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1935)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3176)
	at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30063)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
	at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
	at java.lang.Thread.run(Thread.java:745)



Backend 0:NoSuchColumnFamilyException: org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family  does not exist in region t1,,1435800893075.ef95505bd60b47c29782024f82fbc19d. in table 't1', {NAME => 'cf1', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '1', TTL => 'FOREVER', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}
	at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5546)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1935)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3176)
	at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30063)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
	at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
	at java.lang.Thread.run(Thread.java:745)

CAUSED BY: RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family  does not exist in region t1,,1435800893075.ef95505bd60b47c29782024f82fbc19d. in table 't1', {NAME => 'cf1', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '1', TTL => 'FOREVER', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}
	at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5546)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1935)
	at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3176)
	at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:30063)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
	at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
	at java.lang.Thread.run(Thread.java:745)