diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6de6ed6..db92e71 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -734,9 +734,16 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Defaults to all permissions for the hiveserver2/metastore process user."), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order", "List of comma separated metastore object types that should be pinned in the cache"), - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", new StringSet("BONECP", "DBCP", + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("BONECP", "DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), + METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS("datanucleus.connectionPool.maxPoolSize", 10, + "Specify the maximum number of connections in the connection pool. Note: The configured size will be used by" + + " 2 connection pools (TxnHandler and ObjectStore). When configuring the max connection pool size, it is " + + "recommended to take into account the number of metastore instances and the number of HiveServer2 instances " + + "configured with embedded metastore. To get optimal performance, set config to meet the following condition"+ + "(2 * pool_size * metastore_instances + 2 * pool_size * HS2_instances_with_embedded_metastore) = " + + "(2 * physical_core_count + hard_disk_count)."), // Workaround for DN bug on Postgres: // http://www.datanucleus.org/servlet/forum/viewthread_thread,7985_offset METASTORE_DATANUCLEUS_INIT_COL_INFO("datanucleus.rdbms.initializeColumnInfo", "NONE", diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index d378d06..6bc532b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -3139,6 +3139,7 @@ private static synchronized void setupJdbcConnectionPool(HiveConf conf) throws S String passwd = getMetastoreJdbcPasswd(conf); String connectionPooler = conf.getVar( HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); + int maxPoolSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS); if ("bonecp".equals(connectionPooler)) { BoneCPConfig config = new BoneCPConfig(); @@ -3146,8 +3147,8 @@ private static synchronized void setupJdbcConnectionPool(HiveConf conf) throws S //if we are waiting for connection for 60s, something is really wrong //better raise an error than hang forever config.setConnectionTimeoutInMs(60000); - config.setMaxConnectionsPerPartition(10); config.setPartitionCount(1); + config.setMaxConnectionsPerPartition(maxPoolSize); config.setUser(user); config.setPassword(passwd); connPool = new BoneCPDataSource(config); @@ -3162,6 +3163,7 @@ private static synchronized void setupJdbcConnectionPool(HiveConf conf) throws S connPool = new PoolingDataSource(objectPool); } else if ("hikaricp".equals(connectionPooler)) { HikariConfig config = new HikariConfig(); + config.setMaximumPoolSize(maxPoolSize); config.setJdbcUrl(driverUrl); config.setUsername(user); config.setPassword(passwd); diff --git a/pom.xml b/pom.xml index 48345b9..a73fac0 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ 1.1.1 3.3.0 - 2.5.1 + 2.6.1 4.5.2 4.4.4