diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 897fc4efd4c541ee2da30516c933647f5d4a0af4..c249ffdc0570ee427e813f6bb25df6fda45d1ee7 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.net.URI; import java.nio.ByteBuffer; +import java.sql.SQLException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -58,6 +59,7 @@ import javax.jdo.Transaction; import javax.jdo.datastore.DataStoreCache; import javax.jdo.identity.IntIdentity; +import javax.sql.DataSource; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.exception.ExceptionUtils; @@ -121,6 +123,8 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider; +import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; @@ -533,10 +537,29 @@ private static void correctAutoStartMechanism(Configuration conf) { private static synchronized PersistenceManagerFactory getPMF() { if (pmf == null) { - pmf = JDOHelper.getPersistenceManagerFactory(prop); + + HiveConf conf = new HiveConf(ObjectStore.class); + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + if (dsp == null) { + pmf = JDOHelper.getPersistenceManagerFactory(prop); + } else { + try { + DataSource ds = dsp.create(conf); + Map dsProperties = new HashMap<>(); + //Any preexisting datanucleus property should be passed along + dsProperties.putAll(prop); + dsProperties.put("datanucleus.ConnectionFactory", ds); + dsProperties.put("javax.jdo.PersistenceManagerFactoryClass", + "org.datanucleus.api.jdo.JDOPersistenceManagerFactory"); + pmf = JDOHelper.getPersistenceManagerFactory(dsProperties); + } catch (SQLException e) { + LOG.warn("Could not create PersistenceManagerFactory using " + + "connection pool properties, will fall back", e); + pmf = JDOHelper.getPersistenceManagerFactory(prop); + } + } DataStoreCache dsc = pmf.getDataStoreCache(); if (dsc != null) { - HiveConf conf = new HiveConf(ObjectStore.class); String objTypes = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES); LOG.info("Setting MetaStore object pin classes with hive.metastore.cache.pinobjtypes=\"" + objTypes + "\""); if (objTypes != null && objTypes.length() > 0) { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..b060f72da1d978568366e9981722bcb380894ad9 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.jolbox.bonecp.BoneCPConfig; +import com.jolbox.bonecp.BoneCPDataSource; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.Properties; + +/** + * DataSourceProvider for the BoneCP connection pool. + */ +public class BoneCPDataSourceProvider implements DataSourceProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class); + + public static final String BONECP = "bonecp"; + private static final String CONNECTION_TIMEOUT_PROPERTY= "bonecp.connectionTimeoutInMs"; + private static final String PARTITION_COUNT_PROPERTY= "bonecp.partitionCount"; + + @Override + public DataSource create(Configuration hdpConfig) throws SQLException { + + LOG.debug("Creating BoneCP connection pool for the MetaStore"); + + String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig); + String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig); + String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig); + int maxPoolSize = hdpConfig.getInt( + HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS.varname, + HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS.defaultIntVal); + + Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP); + String connectionTimeout = properties.getProperty(CONNECTION_TIMEOUT_PROPERTY, "30000"); + String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1"); + + BoneCPConfig config = null; + try { + config = new BoneCPConfig(properties); + } catch (Exception e) { + throw new SQLException("Cannot create BoneCP configuration: ", e); + } + config.setJdbcUrl(driverUrl); + //if we are waiting for connection for a long time, something is really wrong + //better raise an error than hang forever + //see DefaultConnectionStrategy.getConnectionInternal() + config.setConnectionTimeoutInMs(Long.parseLong(connectionTimeout)); + config.setMaxConnectionsPerPartition(maxPoolSize); + config.setPartitionCount(Integer.parseInt(partitionCount)); + config.setUser(user); + config.setPassword(passwd); + return new BoneCPDataSource(config); + } + + @Override + public boolean mayReturnClosedConnection() { + // See HIVE-11915 for details + return true; + } + + @Override + public boolean supports(Configuration configuration) { + String poolingType = + configuration.get( + HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE.varname).toLowerCase(); + if (BONECP.equals(poolingType)) { + int boneCpPropsNr = DataSourceProvider.getPrefixedProperties(configuration, BONECP).size(); + LOG.debug("Found " + boneCpPropsNr + " nr. of bonecp specific configurations"); + return boneCpPropsNr > 0; + } + LOG.debug("Configuration requested " + poolingType + " pooling, BoneCpDSProvider exiting"); + return false; + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..a397b1f132d3c2265f95f2f659ec85c7ce97fbcc --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.google.common.collect.Iterables; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.shims.ShimLoader; + +import javax.sql.DataSource; +import java.io.IOException; +import java.sql.SQLException; +import java.util.Properties; + +public interface DataSourceProvider { + + /** + * @param hdpConfig + * @return the new connection pool + */ + DataSource create(Configuration hdpConfig) throws SQLException; + + /** + * BoneCp has a bug which causes closed connections to be returned to the pool + * under certain conditions. (HIVE-11915) + * @return true if the factory creates BoneCp pools which need "special attention" + */ + boolean mayReturnClosedConnection(); + + /** + * @param configuration Hadoop configuration object + * @return factory able to create a connection pool for the implementation + * specified in the configuration + */ + boolean supports(Configuration configuration); + + /** + * @param hdpConfig + * @return subset of properties prefixed by a connection pool specific substring + */ + static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) { + Properties dataSourceProps = new Properties(); + Iterables.filter( + hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix))) + .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue())); + return dataSourceProps; + } + + static String getMetastoreJdbcUser(Configuration conf) { + return conf.get(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME.varname); + } + + static String getMetastoreJdbcPasswd(Configuration conf) throws SQLException { + try { + return ShimLoader.getHadoopShims().getPassword(conf, + HiveConf.ConfVars.METASTOREPWD.varname); + } catch (IOException err) { + throw new SQLException("Error getting metastore password", err); + } + } + + static String getMetastoreJdbcDriverUrl(Configuration conf) throws SQLException { + return conf.get(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname); + } + +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..fa6bb1cc5252a71ccc6f011d5dd062a952de5b8a --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.conf.Configuration; + +/** + * Create a DataSourceProvider for a connectionPool configured in a hadoop + * Configuration object. + */ +public abstract class DataSourceProviderFactory { + + private static final ImmutableList FACTORIES = + ImmutableList.builder().add(new BoneCPDataSourceProvider()).build(); + + /** + * @param hdpConfig hadoop configuration + * @return factory for the configured datanucleus.connectionPoolingType + */ + public static DataSourceProvider getDataSourceProvider(Configuration hdpConfig) { + + for (DataSourceProvider factory : FACTORIES) { + + if (factory.supports(hdpConfig)) { + return factory; + } + } + return null; + } + +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..86d6a26e06897f43a87eae1c46809202e0c7e2f8 --- /dev/null +++ metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * DataSource providers that can construct a connection pool from configuration + * properties in a hadoop configuration object. + */ +package org.apache.hadoop.hive.metastore.datasource; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index b722af6ceb8375c76c3a393cee79ed2387555b69..d28959f3b94c20463fc3223d5e81e506632b89c3 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.metastore.txn; import com.google.common.annotations.VisibleForTesting; -import com.jolbox.bonecp.BoneCPConfig; -import com.jolbox.bonecp.BoneCPDataSource; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -34,23 +32,22 @@ import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.HouseKeeperService; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.datasource.BoneCPDataSourceProvider; +import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.dbcp.PoolingDataSource; -import org.apache.commons.pool.ObjectPool; import org.apache.commons.pool.impl.GenericObjectPool; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.StringableMap; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.util.StringUtils; import javax.sql.DataSource; -import java.io.IOException; import java.io.PrintWriter; import java.nio.ByteBuffer; import java.sql.*; @@ -3168,25 +3165,15 @@ public void countOpenTxns() throws MetaException { } private static synchronized DataSource setupJdbcConnectionPool(HiveConf conf, int maxPoolSize, long getConnectionTimeoutMs) throws SQLException { - String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY); - String user = getMetastoreJdbcUser(conf); - String passwd = getMetastoreJdbcPasswd(conf); + String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(conf); + String user = DataSourceProvider.getMetastoreJdbcUser(conf); + String passwd = DataSourceProvider.getMetastoreJdbcPasswd(conf); String connectionPooler = conf.getVar( HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase(); if ("bonecp".equals(connectionPooler)) { - BoneCPConfig config = new BoneCPConfig(); - config.setJdbcUrl(driverUrl); - //if we are waiting for connection for a long time, something is really wrong - //better raise an error than hang forever - //see DefaultConnectionStrategy.getConnectionInternal() - config.setConnectionTimeoutInMs(getConnectionTimeoutMs); - config.setMaxConnectionsPerPartition(maxPoolSize); - config.setPartitionCount(1); - config.setUser(user); - config.setPassword(passwd); doRetryOnConnPool = true; // Enable retries to work around BONECP bug. - return new BoneCPDataSource(config); + return new BoneCPDataSourceProvider().create(conf); } else if ("dbcp".equals(connectionPooler)) { GenericObjectPool objectPool = new GenericObjectPool(); //https://commons.apache.org/proper/commons-pool/api-1.6/org/apache/commons/pool/impl/GenericObjectPool.html#setMaxActive(int) @@ -3695,19 +3682,6 @@ private String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaE } } - private static String getMetastoreJdbcUser(HiveConf conf) { - return conf.getVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME); - } - - private static String getMetastoreJdbcPasswd(HiveConf conf) throws SQLException { - try { - return ShimLoader.getHadoopShims().getPassword(conf, - HiveConf.ConfVars.METASTOREPWD.varname); - } catch (IOException err) { - throw new SQLException("Error getting metastore password", err); - } - } - private static class NoPoolConnectionPool implements DataSource { // Note that this depends on the fact that no-one in this class calls anything but // getConnection. If you want to use any of the Logger or wrap calls you'll have to @@ -3725,8 +3699,8 @@ public NoPoolConnectionPool(HiveConf conf) { @Override public Connection getConnection() throws SQLException { if (user == null) { - user = getMetastoreJdbcUser(conf); - passwd = getMetastoreJdbcPasswd(conf); + user = DataSourceProvider.getMetastoreJdbcUser(conf); + passwd = DataSourceProvider.getMetastoreJdbcPasswd(conf); } return getConnection(user, passwd); } diff --git metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..628460482dc646f8a38f607e815eddaa3cc2a831 --- /dev/null +++ metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.datasource; + +import com.jolbox.bonecp.BoneCPDataSource; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.sql.DataSource; +import java.sql.SQLException; + +public class TestDataSourceProviderFactory { + + private HiveConf conf; + + @Before + public void init() { + conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "dummyUser"); + conf.setVar(HiveConf.ConfVars.METASTOREPWD, "dummyPass"); + } + + @Test + public void testNoDataSourceCreatedWithoutProps() throws SQLException { + + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNull(dsp); + + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); + + dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNull(dsp); + } + + @Test + public void testCreateBoneCpDataSource() throws SQLException { + + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); + conf.set(BoneCPDataSourceProvider.BONECP + ".firstProp", "value"); + conf.set(BoneCPDataSourceProvider.BONECP + ".secondProp", "value"); + + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof BoneCPDataSource); + } + + @Test + public void testSetBoneCpStringProperty() throws SQLException { + + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); + conf.set(BoneCPDataSourceProvider.BONECP + ".initSQL", "select 1 from dual"); + + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof BoneCPDataSource); + Assert.assertEquals("select 1 from dual", ((BoneCPDataSource)ds).getInitSQL()); + } + + @Test + public void testSetBoneCpNumberProperty() throws SQLException { + + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); + conf.set(BoneCPDataSourceProvider.BONECP + ".acquireRetryDelayInMs", "599"); + + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof BoneCPDataSource); + Assert.assertEquals(599L, ((BoneCPDataSource)ds).getAcquireRetryDelayInMs()); + } + + @Test + public void testSetBoneCpBooleanProperty() throws SQLException { + + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP); + conf.set(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true"); + + DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof BoneCPDataSource); + Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX()); + } +}