diff --git a/binary-package-licenses/README b/binary-package-licenses/README index db6b7fef20..02a4d1106b 100644 --- a/binary-package-licenses/README +++ b/binary-package-licenses/README @@ -19,7 +19,6 @@ ant* apache-curator avatica* avro -bonecp calcite* classmate commons* diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 16bae920df..e3ddbf197b 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1005,7 +1005,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal * @deprecated Use MetastoreConf.CONNECTION_POOLING_TYPE */ @Deprecated - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("BONECP", "DBCP", + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), /** @@ -4831,7 +4831,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.spark.client.rpc.server.address," + "hive.spark.client.rpc.server.port," + "hive.spark.client.rpc.sasl.mechanisms," + - "bonecp.," + "hive.druid.broker.address.default," + "hive.druid.coordinator.address.default," + "hikaricp.," + diff --git a/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm b/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm index 65e6c81957..b8604809d2 100644 --- a/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm +++ b/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm @@ -424,9 +424,6 @@ ($$) /jdo-api-.*\.jar$/ && do { $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; }; - /bonecp-.*\.jar$/ && do { - $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; - }; /commons-pool-.*\.jar$/ && do { $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; }; diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java index 596c3d6fc1..d5641d983e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java @@ -94,7 +94,6 @@ public static void startServices() throws Exception { addToExpectedRestrictedMap("hive.spark.client.rpc.server.port"); addToExpectedRestrictedMap("hive.spark.client.rpc.sasl.mechanisms"); addToExpectedRestrictedMap("hive.query.max.length"); - addToExpectedRestrictedMap("bonecp.test"); addToExpectedRestrictedMap("hive.druid.broker.address.default"); addToExpectedRestrictedMap("hive.druid.coordinator.address.default"); addToExpectedRestrictedMap("hikaricp.test"); diff --git a/metastore/pom.xml b/metastore/pom.xml index 95e2d07c11..c82dc2621a 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -75,16 +75,6 @@ - - com.jolbox - bonecp - ${bonecp.version} - - - com.zaxxer - HikariCP - ${hikaricp.version} - commons-cli commons-cli diff --git a/pom.xml b/pom.xml index 90e39702a1..2322957cd2 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,6 @@ 0.10.0 1.12.0 1.8.2 - 0.8.0.RELEASE 1.21.0 4.2.4 4.1.17 @@ -289,11 +288,6 @@ JavaEWAH ${javaewah.version} - - com.jolbox - bonecp - ${bonecp.version} - com.zaxxer HikariCP diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml index 449d521634..69cdca44db 100644 --- a/standalone-metastore/metastore-common/pom.xml +++ b/standalone-metastore/metastore-common/pom.xml @@ -64,10 +64,6 @@ com.google.protobuf protobuf-java - - com.jolbox - bonecp - com.zaxxer HikariCP diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 4be6bccd61..a874121e12 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -458,7 +458,7 @@ public static ConfVars getMetaConf(String name) { "To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.\n" + "For example, jdbc:postgresql://myhost/db?ssl=true for postgres database."), CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", - "datanucleus.connectionPoolingType", "HikariCP", new StringSetValidator("BONECP", "DBCP", + "datanucleus.connectionPoolingType", "HikariCP", new StringSetValidator("DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 49133271c4..e492ce1070 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -66,10 +66,6 @@ com.google.protobuf protobuf-java - - com.jolbox - bonecp - com.zaxxer HikariCP diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java deleted file mode 100644 index de202c9bfc..0000000000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.datasource; - -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import javax.sql.DataSource; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.MetricSet; -import com.jolbox.bonecp.BoneCPConfig; -import com.jolbox.bonecp.BoneCPDataSource; -import com.jolbox.bonecp.StatisticsMBean; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.DatabaseProduct; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.metrics.Metrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.hive.metastore.DatabaseProduct.MYSQL; -import static org.apache.hadoop.hive.metastore.DatabaseProduct.determineDatabaseProduct; - -/** - * DataSourceProvider for the BoneCP connection pool. - */ -public class BoneCPDataSourceProvider implements DataSourceProvider { - - private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class); - - public static final String BONECP = "bonecp"; - private static final String CONNECTION_TIMEOUT_PROPERTY= BONECP + ".connectionTimeoutInMs"; - private static final String PARTITION_COUNT_PROPERTY= BONECP + ".partitionCount"; - - @Override - public DataSource create(Configuration hdpConfig) throws SQLException { - LOG.debug("Creating BoneCP connection pool for the MetaStore"); - - String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig); - String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig); - String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig); - - int maxPoolSize = MetastoreConf.getIntVar(hdpConfig, - MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS); - - Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP); - long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L); - String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1"); - - BoneCPConfig config; - try { - config = new BoneCPConfig(properties); - } catch (Exception e) { - throw new SQLException("Cannot create BoneCP configuration: ", e); - } - config.setJdbcUrl(driverUrl); - config.setUser(user); - config.setPassword(passwd); - // if we are waiting for connection for a long time, something is really wrong - // better raise an error than hang forever - // see DefaultConnectionStrategy.getConnectionInternal() - config.setConnectionTimeoutInMs(connectionTimeout); - config.setMaxConnectionsPerPartition(maxPoolSize); - config.setPartitionCount(Integer.parseInt(partitionCount)); - - Properties connProperties = new Properties(); - - DatabaseProduct dbProduct = determineDatabaseProduct(driverUrl); - switch (dbProduct){ - case MYSQL: - connProperties.put("allowMultiQueries", true); - connProperties.put("rewriteBatchedStatements", true); - break; - case POSTGRES: - connProperties.put("reWriteBatchedInserts", true); - break; - } - config.setDriverProperties(connProperties); - - if (dbProduct == MYSQL) { - config.setInitSQL("SET @@session.sql_mode=ANSI_QUOTES"); - } - return initMetrics(new BoneCPDataSource(config)); - } - - @Override - public boolean mayReturnClosedConnection() { - // See HIVE-11915 for details - return true; - } - - @Override - public String getPoolingType() { - return BONECP; - } - - private BoneCPDataSource initMetrics(BoneCPDataSource ds) { - final MetricRegistry registry = Metrics.getRegistry(); - if (registry != null) { - registry.registerAll(new BoneCPMetrics(ds)); - } - return ds; - } - - private static class BoneCPMetrics implements MetricSet { - private BoneCPDataSource ds; - private Optional poolName; - - private BoneCPMetrics(final BoneCPDataSource ds) { - this.ds = ds; - this.poolName = Optional.ofNullable(ds.getPoolName()); - } - - private String name(final String gaugeName) { - return poolName.orElse("BoneCP") + ".pool." + gaugeName; - } - - @Override - public Map getMetrics() { - final Map gauges = new HashMap<>(); - - gauges.put(name("TotalConnections"), new Gauge() { - @Override - public Integer getValue() { - if (ds.getPool() != null) { - return ds.getPool().getStatistics().getTotalCreatedConnections(); - } else { - return 0; - } - } - }); - - gauges.put(name("IdleConnections"), new Gauge() { - @Override - public Integer getValue() { - if (ds.getPool() != null) { - return ds.getPool().getStatistics().getTotalFree(); - } else { - return 0; - } - } - }); - - gauges.put(name("ActiveConnections"), new Gauge() { - @Override - public Integer getValue() { - if (ds.getPool() != null) { - return ds.getPool().getStatistics().getTotalLeased(); - } else { - return 0; - } - } - }); - - gauges.put(name("WaitTimeAvg"), new Gauge() { - @Override - public Double getValue() { - if (ds.getPool() != null) { - return ds.getPool().getStatistics().getConnectionWaitTimeAvg(); - } else { - return 0.0; - } - } - }); - - return Collections.unmodifiableMap(gauges); - } - } - -} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java index d71fa7d4ea..29633b500d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java @@ -34,13 +34,6 @@ */ DataSource create(Configuration hdpConfig) throws SQLException; - /** - * BoneCp has a bug which causes closed connections to be returned to the pool - * under certain conditions. (HIVE-11915) - * @return true if the factory creates BoneCp pools which need "special attention" - */ - boolean mayReturnClosedConnection(); - /** * Get the declared pooling type string. This is used to check against the constant in * config options. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java index bad12b2341..f2a61842dc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java @@ -29,7 +29,6 @@ public abstract class DataSourceProviderFactory { private static final ImmutableList FACTORIES = ImmutableList.of( new HikariCPDataSourceProvider(), - new BoneCPDataSourceProvider(), new DbCPDataSourceProvider()); /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java index 12cacd2bed..c687a252b8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DbCPDataSourceProvider.java @@ -126,12 +126,6 @@ public DataSource create(Configuration hdpConfig) throws SQLException { return new PoolingDataSource(objectPool); } - @Override - public boolean mayReturnClosedConnection() { - // Only BoneCP should return true - return false; - } - @Override public String getPoolingType() { return DBCP; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java index 3e871e9fba..8a4a9c2cbb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java @@ -86,12 +86,6 @@ public DataSource create(Configuration hdpConfig) throws SQLException { return new HikariDataSource(initMetrics(config)); } - @Override - public boolean mayReturnClosedConnection() { - // Only BoneCP should return true - return false; - } - @Override public String getPoolingType() { return HIKARI; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index d080df417b..e7910c1c5d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -185,7 +185,6 @@ private static DataSource connPool; private static DataSource connPoolMutex; - private static boolean doRetryOnConnPool = false; // Query definitions private static final String HIVE_LOCKS_INSERT_QRY = "INSERT INTO \"HIVE_LOCKS\" ( " + @@ -2304,9 +2303,7 @@ public long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long t /** * As much as possible (i.e. in absence of retries) we want both operations to be done on the same - * connection (but separate transactions). This avoid some flakiness in BONECP where if you - * perform an operation on 1 connection and immediately get another from the pool, the 2nd one - * doesn't see results of the first. + * connection (but separate transactions). * * Retry-by-caller note: If the call to lock is from a transaction, then in the worst case * there will be a duplicate set of locks but both sets will belong to the same txn so they @@ -3772,21 +3769,17 @@ public long setTimeout(long milliseconds) { Connection getDbConn(int isolationLevel) throws SQLException { return getDbConn(isolationLevel, connPool); } + private Connection getDbConn(int isolationLevel, DataSource connPool) throws SQLException { - int rc = doRetryOnConnPool ? 10 : 1; Connection dbConn = null; - while (true) { - try { - dbConn = connPool.getConnection(); - dbConn.setAutoCommit(false); - dbConn.setTransactionIsolation(isolationLevel); - return dbConn; - } catch (SQLException e){ - closeDbConn(dbConn); - if ((--rc) <= 0) throw e; - LOG.error("There is a problem with a connection from the pool, retrying(rc=" + rc + "): " + - getMessage(e), e); - } + try { + dbConn = connPool.getConnection(); + dbConn.setAutoCommit(false); + dbConn.setTransactionIsolation(isolationLevel); + return dbConn; + } catch (SQLException e) { + closeDbConn(dbConn); + throw e; } } @@ -4943,7 +4936,6 @@ public void countOpenTxns() throws MetaException { private static synchronized DataSource setupJdbcConnectionPool(Configuration conf, int maxPoolSize, long getConnectionTimeoutMs) throws SQLException { DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); if (dsp != null) { - doRetryOnConnPool = dsp.mayReturnClosedConnection(); return dsp.create(conf); } else { String connectionPooler = MetastoreConf.getVar(conf, ConfVars.CONNECTION_POOLING_TYPE).toLowerCase(); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java index f3b6f9f2da..43ca0ef433 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.metastore.datasource; -import com.jolbox.bonecp.BoneCPDataSource; import com.zaxxer.hikari.HikariDataSource; import org.apache.commons.dbcp.PoolingDataSource; import org.apache.hadoop.conf.Configuration; @@ -54,60 +53,6 @@ public void testNoDataSourceCreatedWithoutProps() throws SQLException { Assert.assertNull(dsp); } - @Test - public void testCreateBoneCpDataSource() throws SQLException { - - MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); - - DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); - Assert.assertNotNull(dsp); - - DataSource ds = dsp.create(conf); - Assert.assertTrue(ds instanceof BoneCPDataSource); - } - - @Test - public void testSetBoneCpStringProperty() throws SQLException { - - MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); - conf.set(BoneCPDataSourceProvider.BONECP + ".initSQL", "select 1 from dual"); - - DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); - Assert.assertNotNull(dsp); - - DataSource ds = dsp.create(conf); - Assert.assertTrue(ds instanceof BoneCPDataSource); - Assert.assertEquals("select 1 from dual", ((BoneCPDataSource)ds).getInitSQL()); - } - - @Test - public void testSetBoneCpNumberProperty() throws SQLException { - - MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); - conf.set(BoneCPDataSourceProvider.BONECP + ".acquireRetryDelayInMs", "599"); - - DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); - Assert.assertNotNull(dsp); - - DataSource ds = dsp.create(conf); - Assert.assertTrue(ds instanceof BoneCPDataSource); - Assert.assertEquals(599L, ((BoneCPDataSource)ds).getAcquireRetryDelayInMs()); - } - - @Test - public void testSetBoneCpBooleanProperty() throws SQLException { - - MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); - conf.set(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true"); - - DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); - Assert.assertNotNull(dsp); - - DataSource ds = dsp.create(conf); - Assert.assertTrue(ds instanceof BoneCPDataSource); - Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX()); - } - @Test public void testCreateHikariCpDataSource() throws SQLException { diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index b26e3b3c99..516ce4a40a 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -68,7 +68,6 @@ 3.5.2 - 0.8.0.RELEASE 1.5.7 1.4 3.9 @@ -141,11 +140,6 @@ protobuf-java ${protobuf.version} - - com.jolbox - bonecp - ${bonecp.version} - com.zaxxer HikariCP