diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 103975fd85..fe9eac9ae5 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -431,12 +431,6 @@ ${plexus.version} test - - commons-dbcp - commons-dbcp - ${commons-dbcp.version} - test - diff --git a/jdbc-handler/pom.xml b/jdbc-handler/pom.xml index b2d9ec8eb9..da137cdbed 100644 --- a/jdbc-handler/pom.xml +++ b/jdbc-handler/pom.xml @@ -89,7 +89,6 @@ org.apache.commons commons-dbcp2 - ${commons-dbcp2.version} diff --git a/metastore/pom.xml b/metastore/pom.xml index cca56a57f6..f1d5b66a05 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -86,16 +86,6 @@ derby ${derby.version} - - commons-pool - commons-pool - ${commons-pool.version} - - - commons-dbcp - commons-dbcp - ${commons-dbcp.version} - org.antlr antlr-runtime diff --git a/pom.xml b/pom.xml index 29cde34b74..25e78ff5d6 100644 --- a/pom.xml +++ b/pom.xml @@ -141,10 +141,7 @@ 1.1 2.6 3.9 - 1.5.4 - 1.4 - 2.6.1 - 2.6.0 + 2.7.0 1.8 10.14.1.0 3.1.0 @@ -350,6 +347,11 @@ commons-io ${commons-io.version} + + org.apache.commons + commons-dbcp2 + ${commons-dbcp2.version} + io.netty netty-all diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml index ab269156a7..e8748c7a5f 100644 --- a/standalone-metastore/metastore-common/pom.xml +++ b/standalone-metastore/metastore-common/pom.xml @@ -68,10 +68,6 @@ com.zaxxer HikariCP - - commons-dbcp - commons-dbcp - io.dropwizard.metrics metrics-core diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 6126cae118..982ba1c800 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -71,8 +71,8 @@ HikariCP - commons-dbcp - commons-dbcp + org.apache.commons + commons-dbcp2 io.dropwizard.metrics diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java index c687a252b8..1a5a1d2384 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java @@ -17,24 +17,26 @@ */ package org.apache.hadoop.hive.metastore.datasource; -import org.apache.commons.dbcp.BasicDataSource; -import org.apache.commons.dbcp.ConnectionFactory; -import org.apache.commons.dbcp.DataSourceConnectionFactory; -import org.apache.commons.dbcp.PoolableConnectionFactory; -import org.apache.commons.dbcp.PoolingDataSource; -import org.apache.commons.pool.impl.GenericObjectPool; +import static org.apache.hadoop.hive.metastore.DatabaseProduct.MYSQL; +import static org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct; + +import java.sql.SQLException; + +import javax.sql.DataSource; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.dbcp2.ConnectionFactory; +import org.apache.commons.dbcp2.DataSourceConnectionFactory; +import org.apache.commons.dbcp2.PoolableConnectionFactory; +import org.apache.commons.dbcp2.PoolingDataSource; +import org.apache.commons.pool2.impl.BaseObjectPoolConfig; +import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.sql.DataSource; -import java.sql.SQLException; - -import static org.apache.hadoop.hive.metastore.DatabaseProduct.MYSQL; -import static org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct; - /** * DataSourceProvider for the dbcp connection pool. */ @@ -55,6 +57,7 @@ private static final String CONNECTION_SOFT_MIN_EVICTABLE_IDLE_TIME = DBCP + ".softMinEvictableIdleTimeMillis"; private static final String CONNECTION_LIFO = DBCP + ".lifo"; + @SuppressWarnings({ "rawtypes", "unchecked" }) @Override public DataSource create(Configuration hdpConfig) throws SQLException { LOG.debug("Creating dbcp connection pool for the MetaStore"); @@ -67,6 +70,8 @@ public DataSource create(Configuration hdpConfig) throws SQLException { dbcpDs.setUrl(driverUrl); dbcpDs.setUsername(user); dbcpDs.setPassword(passwd); + dbcpDs.setDefaultReadOnly(false); + dbcpDs.setDefaultAutoCommit(true); DatabaseProduct dbProduct = determineDatabaseProduct(driverUrl); switch (dbProduct){ @@ -77,31 +82,36 @@ public DataSource create(Configuration hdpConfig) throws SQLException { case POSTGRES: dbcpDs.setConnectionProperties("reWriteBatchedInserts=true"); break; + default: + break; } int maxPoolSize = hdpConfig.getInt( MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.getVarname(), ((Long) MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.getDefaultVal()).intValue()); long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L); - int connectionMaxIlde = hdpConfig.getInt(CONNECTION_MAX_IDLE_PROPERTY, GenericObjectPool.DEFAULT_MAX_IDLE); - int connectionMinIlde = hdpConfig.getInt(CONNECTION_MIN_IDLE_PROPERTY, GenericObjectPool.DEFAULT_MIN_IDLE); + int connectionMaxIlde = hdpConfig.getInt(CONNECTION_MAX_IDLE_PROPERTY, 8); + int connectionMinIlde = hdpConfig.getInt(CONNECTION_MIN_IDLE_PROPERTY, 0); boolean testOnBorrow = hdpConfig.getBoolean(CONNECTION_TEST_BORROW_PROPERTY, - GenericObjectPool.DEFAULT_TEST_ON_BORROW); + BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW); long evictionTimeMillis = hdpConfig.getLong(CONNECTION_MIN_EVICT_MILLIS_PROPERTY, - GenericObjectPool.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS); + BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_TIME_MILLIS); boolean testWhileIdle = hdpConfig.getBoolean(CONNECTION_TEST_IDLEPROPERTY, - GenericObjectPool.DEFAULT_TEST_WHILE_IDLE); + BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE); long timeBetweenEvictionRuns = hdpConfig.getLong(CONNECTION_TIME_BETWEEN_EVICTION_RUNS_MILLIS, - GenericObjectPool.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS); + BaseObjectPoolConfig.DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS); int numTestsPerEvictionRun = hdpConfig.getInt(CONNECTION_NUM_TESTS_PER_EVICTION_RUN, - GenericObjectPool.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); - boolean testOnReturn = hdpConfig.getBoolean(CONNECTION_TEST_ON_RETURN, GenericObjectPool.DEFAULT_TEST_ON_RETURN); + BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); + boolean testOnReturn = hdpConfig.getBoolean(CONNECTION_TEST_ON_RETURN, BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN); long softMinEvictableIdleTimeMillis = hdpConfig.getLong(CONNECTION_SOFT_MIN_EVICTABLE_IDLE_TIME, - GenericObjectPool.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS); - boolean lifo = hdpConfig.getBoolean(CONNECTION_LIFO, GenericObjectPool.DEFAULT_LIFO); + BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME_MILLIS); + boolean lifo = hdpConfig.getBoolean(CONNECTION_LIFO, BaseObjectPoolConfig.DEFAULT_LIFO); - GenericObjectPool objectPool = new GenericObjectPool(); - objectPool.setMaxActive(maxPoolSize); - objectPool.setMaxWait(connectionTimeout); + ConnectionFactory connFactory = new DataSourceConnectionFactory(dbcpDs); + PoolableConnectionFactory poolableConnFactory = new PoolableConnectionFactory(connFactory, null); + + GenericObjectPool objectPool = new GenericObjectPool(poolableConnFactory); + objectPool.setMaxTotal(maxPoolSize); + objectPool.setMaxWaitMillis(connectionTimeout); objectPool.setMaxIdle(connectionMaxIlde); objectPool.setMinIdle(connectionMinIlde); objectPool.setTestOnBorrow(testOnBorrow); @@ -113,13 +123,6 @@ public DataSource create(Configuration hdpConfig) throws SQLException { objectPool.setSoftMinEvictableIdleTimeMillis(softMinEvictableIdleTimeMillis); objectPool.setLifo(lifo); - ConnectionFactory connFactory = new DataSourceConnectionFactory(dbcpDs); - // This doesn't get used, but it's still necessary, see - // https://git1-us-west.apache.org/repos/asf?p=commons-dbcp.git;a=blob;f=doc/ManualPoolingDataSourceExample.java; - // h=f45af2b8481f030b27364e505984c0eef4f35cdb;hb=refs/heads/DBCP_1_5_x_BRANCH - PoolableConnectionFactory poolableConnFactory = - new PoolableConnectionFactory(connFactory, objectPool, null, null, false, true); - if (dbProduct == MYSQL) { poolableConnFactory.setValidationQuery("SET @@session.sql_mode=ANSI_QUOTES"); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java index 43ca0ef433..33955ed02e 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hive.metastore.datasource; import com.zaxxer.hikari.HikariDataSource; -import org.apache.commons.dbcp.PoolingDataSource; + +import org.apache.commons.dbcp2.PoolingDataSource; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java index 11099a6b9a..709c90ca25 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java @@ -31,7 +31,7 @@ import java.sql.SQLException; import java.util.Random; -import org.apache.commons.dbcp.DelegatingConnection; +import org.apache.commons.dbcp2.DelegatingConnection; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.text.StrTokenizer; import org.apache.hadoop.conf.Configuration; diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 26e82677eb..16ec4ebdf3 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -69,9 +69,9 @@ 3.5.2 1.5.7 - 1.4 3.9 1.1.3 + 2.7.0 4.2.4 4.1.17 3.2.0-m3 @@ -146,11 +146,6 @@ HikariCP ${hikaricp.version} - - commons-dbcp - commons-dbcp - ${commons-dbcp.version} - io.dropwizard.metrics metrics-core @@ -244,6 +239,11 @@ hive-storage-api ${storage-api.version} + + org.apache.commons + commons-dbcp2 + ${commons-dbcp2.version} + org.apache.logging.log4j log4j-slf4j-impl