[HIVE-18592] DP insert on insert only table causes StatTask to fail

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Duplicate
• Component/s: Transactions
• Assignee: Unassigned
• Reporter: Zoltan Haindrich
• Created: 31/Jan/18 10:21
• Updated: 31/Jan/18 15:48
• Resolved: 31/Jan/18 11:06

Description

can be reproduced with:

set hive.mapred.mode=nonstrict;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;


set hive.create.as.insert.only=true;
set metastore.create.as.acid=true;

drop table if exists student;

create table student(
name string,
age int,
gpa double);

insert into student values
        ('asd',1,2),
        ('asdx',2,3),
        ('asdx',2,3),
        ('asdx',3,3),
        ('asdx',3,3),
        ('asdx',3,3);

create table p1 (name STRING, GPA DOUBLE) PARTITIONED BY (age INT);

SET hive.exec.dynamic.partition.mode=nonstrict;

INSERT OVERWRITE TABLE p1 PARTITION (age) SELECT name, gpa, age FROM student;

causes exception

2018-01-31T02:16:24,135 ERROR [22bd4065-6e2f-4f4c-8f29-8d6aad8edda8 main] exec.StatsTask: Failed to run stats task
org.apache.hadoop.hive.ql.metadata.HiveException: NoSuchObjectException(message:Partition for which stats is gathered doesn't exist.)
        at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:4295) ~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.persistColumnStats(ColStatsProcessor.java:180) ~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.process(ColStatsProcessor.java:84) ~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:108) [hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205) [hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97) [hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
...
Caused by: org.apache.hadoop.hive.metastore.api.NoSuchObjectException: Partition for which stats is gathered doesn't exist.
        at org.apache.hadoop.hive.metastore.ObjectStore.updatePartitionColumnStatistics(ObjectStore.java:7757) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_151]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_151]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_151]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_151]
        at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy38.updatePartitionColumnStatistics(Unknown Source) ~[?:?]
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.updatePartitonColStats(HiveMetaStore.java:5394) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.set_aggr_stats_for(HiveMetaStore.java:6907) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_151]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_151]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_151]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_151]
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy40.set_aggr_stats_for(Unknown Source) ~[?:?]
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.setPartitionColumnStatistics(HiveMetaStoreClient.java:1736) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.setPartitionColumnStatistics(SessionHiveMetaStoreClient.java:375) ~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_151]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_151]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_151]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_151]
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:212) ~[hive-standalone-metastore-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy41.setPartitionColumnStatistics(Unknown Source) ~[?:?]
        at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:4292) ~[hive-exec-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]

Issue Links

• duplicates: HIVE-18237 missing results for insert_only table after DP insert