[HIVE-26040] Fix DirectSqlUpdateStat.getNextCSIdForMPartitionColumnStatistics for mssql

• Type: Sub-task
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 4.0.0-alpha-1
• Labels: pull-request-available
• Assignee: Peter Vary
• Reporter: Peter Vary
• Created: 16/Mar/22 18:42
• Updated: 17/Nov/22 08:48
• Resolved: 17/Mar/22 11:25

Description

mvn test -Dtest=TestMiniLlapLocalCliDriver -Dqfile=list_bucket_dml_9.q -Dtest.metastore.db=mssql

Fails with

2022-03-15T07:57:17,078 ERROR [2b933b88-6083-4750-b151-2d2c7e04ccce main] metastore.DirectSqlUpdateStat: Unable to getNextCSIdForMPartitionColumnStatistics
com.microsoft.sqlserver.jdbc.SQLServerException: Line 1: FOR UPDATE clause allowed only for DECLARE CURSOR.
        at com.microsoft.sqlserver.jdbc.SQLServerException.makeFromDatabaseError(SQLServerException.java:258) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement.getNextResult(SQLServerStatement.java:1535) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement.doExecuteStatement(SQLServerStatement.java:845) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement$StmtExecCmd.doExecute(SQLServerStatement.java:752) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.TDSCommand.execute(IOBuffer.java:7151) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerConnection.executeCommand(SQLServerConnection.java:2478) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement.executeCommand(SQLServerStatement.java:219) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement.executeStatement(SQLServerStatement.java:199) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.microsoft.sqlserver.jdbc.SQLServerStatement.executeQuery(SQLServerStatement.java:654) ~[mssql-jdbc-6.2.1.jre8.jar:?]
        at com.zaxxer.hikari.pool.ProxyStatement.executeQuery(ProxyStatement.java:108) ~[HikariCP-2.6.1.jar:?]
        at com.zaxxer.hikari.pool.HikariProxyStatement.executeQuery(HikariProxyStatement.java) ~[HikariCP-2.6.1.jar:?]
        at org.apache.hadoop.hive.metastore.DirectSqlUpdateStat.getNextCSIdForMPartitionColumnStatistics(DirectSqlUpdateStat.java:676) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.MetaStoreDirectSql.updatePartitionColumnStatisticsBatch(MetaStoreDirectSql.java:2966) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.ObjectStore.updatePartitionColumnStatisticsInBatch(ObjectStore.java:9849) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_261]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_261]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_261]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_261]
        at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:97) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy60.updatePartitionColumnStatisticsInBatch(Unknown Source) [?:?]
        at org.apache.hadoop.hive.metastore.HMSHandler.updatePartitionColStatsForOneBatch(HMSHandler.java:7060) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.HMSHandler.updatePartitionColStatsInBatch(HMSHandler.java:7113) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.HMSHandler.set_aggr_stats_for(HMSHandler.java:9137) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_261]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_261]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_261]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_261]
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:146) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) [hive-standalone-metastore-server-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy61.set_aggr_stats_for(Unknown Source) [?:?]
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.setPartitionColumnStatistics(HiveMetaStoreClient.java:3186) [hive-standalone-metastore-common-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.setPartitionColumnStatistics(SessionHiveMetaStoreClient.java:572) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_261]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_261]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_261]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_261]
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:218) [hive-standalone-metastore-common-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at com.sun.proxy.$Proxy62.setPartitionColumnStatistics(Unknown Source) [?:?]
        at org.apache.hadoop.hive.ql.metadata.Hive.setPartitionColumnStatistics(Hive.java:5646) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.persistColumnStats(ColStatsProcessor.java:219) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.stats.ColStatsProcessor.process(ColStatsProcessor.java:93) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:107) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:212) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:361) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:334) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:245) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:106) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:348) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:204) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:153) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.Driver.run(Driver.java:148) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:185) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:228) [hive-exec-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:255) [hive-cli-4.0.0-SNAPSHOT.jar:?]
        at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:200) [hive-cli-4.0.0-SNAPSHOT.jar:?]
        at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:126) [hive-cli-4.0.0-SNAPSHOT.jar:?]
        at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:421) [hive-cli-4.0.0-SNAPSHOT.jar:?]
        at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:352) [hive-cli-4.0.0-SNAPSHOT.jar:?]
        at org.apache.hadoop.hive.ql.dataset.QTestDatasetHandler.initDataset(QTestDatasetHandler.java:86) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.dataset.QTestDatasetHandler.beforeTest(QTestDatasetHandler.java:190) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.qoption.QTestOptionDispatcher.beforeTest(QTestOptionDispatcher.java:79) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.ql.QTestUtil.cliInit(QTestUtil.java:601) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:111) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:157) [hive-it-util-4.0.0-SNAPSHOT.jar:4.0.0-SNAPSHOT]
        at org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:62) [test-classes/:?]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_261]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_261]

Issue Links

• relates to: HIVE-25540 Enable batch update of column stats only for MySql and Postgres
• links to: GitHub Pull Request #3112