Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java =================================================================== --- /dev/null +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -0,0 +1,507 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.List; +import java.util.Map; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.PrivilegeBag; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.metastore.model.MDBPrivilege; +import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; +import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; +import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; +import org.apache.hadoop.hive.metastore.model.MRoleMap; +import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; +import org.apache.hadoop.hive.metastore.model.MTablePrivilege; + +/** + * + * DummyRawStoreForJdoConnection. + * + * An implementation of RawStore that verifies the DummyJdoConnectionUrlHook has already been + * applied when this class's setConf method is called, by checking that the value of the + * METASTORECONNECTURLKEY ConfVar has been updated. + * + * All non-void methods return default values. + */ +public class DummyRawStoreForJdoConnection implements RawStore { + + @Override + public Configuration getConf() { + + return null; + } + + @Override + public void setConf(Configuration arg0) { + String expected = DummyJdoConnectionUrlHook.newUrl; + String actual = arg0.get(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname); + + Assert.assertEquals("The expected URL used by JDO to connect to the metastore: " + expected + + " did not match the actual value when the Raw Store was initialized: " + actual, + expected, actual); + } + + @Override + public void shutdown() { + + + } + + @Override + public boolean openTransaction() { + + return false; + } + + @Override + public boolean commitTransaction() { + + return false; + } + + @Override + public void rollbackTransaction() { + + + } + + @Override + public void createDatabase(Database db) throws InvalidObjectException, MetaException { + + + } + + @Override + public Database getDatabase(String name) throws NoSuchObjectException { + + return null; + } + + @Override + public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException { + + return false; + } + + @Override + public boolean alterDatabase(String dbname, Database db) throws NoSuchObjectException, + MetaException { + + return false; + } + + @Override + public List getDatabases(String pattern) throws MetaException { + + return null; + } + + @Override + public List getAllDatabases() throws MetaException { + + return null; + } + + @Override + public boolean createType(Type type) { + + return false; + } + + @Override + public Type getType(String typeName) { + + return null; + } + + @Override + public boolean dropType(String typeName) { + + return false; + } + + @Override + public void createTable(Table tbl) throws InvalidObjectException, MetaException { + + + } + + @Override + public boolean dropTable(String dbName, String tableName) throws MetaException { + + return false; + } + + @Override + public Table getTable(String dbName, String tableName) throws MetaException { + + return null; + } + + @Override + public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { + + return false; + } + + @Override + public Partition getPartition(String dbName, String tableName, List part_vals) + throws MetaException, NoSuchObjectException { + + return null; + } + + @Override + public boolean dropPartition(String dbName, String tableName, List part_vals) + throws MetaException { + + return false; + } + + @Override + public List getPartitions(String dbName, String tableName, int max) + throws MetaException { + + return null; + } + + @Override + public void alterTable(String dbname, String name, Table newTable) throws InvalidObjectException, + MetaException { + + + } + + @Override + public List getTables(String dbName, String pattern) throws MetaException { + + return null; + } + + @Override + public List getTableObjectsByName(String dbname, List tableNames) + throws MetaException, UnknownDBException { + + return null; + } + + @Override + public List getAllTables(String dbName) throws MetaException { + + return null; + } + + @Override + public List listTableNamesByFilter(String dbName, String filter, short max_tables) + throws MetaException, UnknownDBException { + + return null; + } + + @Override + public List listPartitionNames(String db_name, String tbl_name, short max_parts) + throws MetaException { + + return null; + } + + @Override + public List listPartitionNamesByFilter(String db_name, String tbl_name, String filter, + short max_parts) throws MetaException { + + return null; + } + + @Override + public void alterPartition(String db_name, String tbl_name, List part_vals, + Partition new_part) throws InvalidObjectException, MetaException { + + + } + + @Override + public boolean addIndex(Index index) throws InvalidObjectException, MetaException { + + return false; + } + + @Override + public Index getIndex(String dbName, String origTableName, String indexName) + throws MetaException { + + return null; + } + + @Override + public boolean dropIndex(String dbName, String origTableName, String indexName) + throws MetaException { + + return false; + } + + @Override + public List getIndexes(String dbName, String origTableName, int max) + throws MetaException { + + return null; + } + + @Override + public List listIndexNames(String dbName, String origTableName, short max) + throws MetaException { + + return null; + } + + @Override + public void alterIndex(String dbname, String baseTblName, String name, Index newIndex) + throws InvalidObjectException, MetaException { + + + } + + @Override + public List getPartitionsByFilter(String dbName, String tblName, String filter, + short maxParts) throws MetaException, NoSuchObjectException { + + return null; + } + + @Override + public List getPartitionsByNames(String dbName, String tblName, + List partNames) throws MetaException, NoSuchObjectException { + + return null; + } + + @Override + public Table markPartitionForEvent(String dbName, String tblName, Map partVals, + PartitionEventType evtType) throws MetaException, UnknownTableException, + InvalidPartitionException, UnknownPartitionException { + + return null; + } + + @Override + public boolean isPartitionMarkedForEvent(String dbName, String tblName, + Map partName, PartitionEventType evtType) throws MetaException, + UnknownTableException, InvalidPartitionException, UnknownPartitionException { + + return false; + } + + @Override + public boolean addRole(String rowName, String ownerName) throws InvalidObjectException, + MetaException, NoSuchObjectException { + + return false; + } + + @Override + public boolean removeRole(String roleName) throws MetaException, NoSuchObjectException { + + return false; + } + + @Override + public boolean grantRole(Role role, String userName, PrincipalType principalType, String grantor, + PrincipalType grantorType, boolean grantOption) throws MetaException, NoSuchObjectException, + InvalidObjectException { + + return false; + } + + @Override + public boolean revokeRole(Role role, String userName, PrincipalType principalType) + throws MetaException, NoSuchObjectException { + + return false; + } + + @Override + public PrincipalPrivilegeSet getUserPrivilegeSet(String userName, List groupNames) + throws InvalidObjectException, MetaException { + + return null; + } + + @Override + public PrincipalPrivilegeSet getDBPrivilegeSet(String dbName, String userName, + List groupNames) throws InvalidObjectException, MetaException { + + return null; + } + + @Override + public PrincipalPrivilegeSet getTablePrivilegeSet(String dbName, String tableName, + String userName, List groupNames) throws InvalidObjectException, MetaException { + + return null; + } + + @Override + public PrincipalPrivilegeSet getPartitionPrivilegeSet(String dbName, String tableName, + String partition, String userName, List groupNames) throws InvalidObjectException, + MetaException { + + return null; + } + + @Override + public PrincipalPrivilegeSet getColumnPrivilegeSet(String dbName, String tableName, + String partitionName, String columnName, String userName, List groupNames) + throws InvalidObjectException, MetaException { + + return null; + } + + @Override + public List listPrincipalGlobalGrants(String principalName, + PrincipalType principalType) { + + return null; + } + + @Override + public List listPrincipalDBGrants(String principalName, + PrincipalType principalType, String dbName) { + + return null; + } + + @Override + public List listAllTableGrants(String principalName, + PrincipalType principalType, String dbName, String tableName) { + + return null; + } + + @Override + public List listPrincipalPartitionGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partName) { + + return null; + } + + @Override + public List listPrincipalTableColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String columnName) { + + return null; + } + + @Override + public List listPrincipalPartitionColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partName, + String columnName) { + + return null; + } + + @Override + public boolean grantPrivileges(PrivilegeBag privileges) throws InvalidObjectException, + MetaException, NoSuchObjectException { + + return false; + } + + @Override + public boolean revokePrivileges(PrivilegeBag privileges) throws InvalidObjectException, + MetaException, NoSuchObjectException { + + return false; + } + + @Override + public Role getRole(String roleName) throws NoSuchObjectException { + + return null; + } + + @Override + public List listRoleNames() { + + return null; + } + + @Override + public List listRoles(String principalName, PrincipalType principalType) { + + return null; + } + + @Override + public Partition getPartitionWithAuth(String dbName, String tblName, List partVals, + String user_name, List group_names) throws MetaException, NoSuchObjectException, + InvalidObjectException { + + return null; + } + + @Override + public List getPartitionsWithAuth(String dbName, String tblName, short maxParts, + String userName, List groupNames) throws MetaException, NoSuchObjectException, + InvalidObjectException { + + return null; + } + + @Override + public List listPartitionNamesPs(String db_name, String tbl_name, List part_vals, + short max_parts) throws MetaException, NoSuchObjectException { + + return null; + } + + @Override + public List listPartitionsPsWithAuth(String db_name, String tbl_name, + List part_vals, short max_parts, String userName, List groupNames) + throws MetaException, InvalidObjectException, NoSuchObjectException { + + return null; + } + + @Override + public long cleanupEvents() { + + return 0; + } + +} Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreConnectionUrlHook.java =================================================================== --- /dev/null +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreConnectionUrlHook.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; + +/** + * TestMetaStoreConnectionUrlHook + * Verifies that when an instance of an implementation of RawStore is initialized, the connection + * URL has already been updated by any metastore connect URL hooks. + */ +public class TestMetaStoreConnectionUrlHook extends TestCase { + private HiveConf hiveConf; + + @Override + protected void setUp() throws Exception { + + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testUrlHook() throws Exception { + hiveConf = new HiveConf(this.getClass()); + hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLHOOK, + "org.apache.hadoop.hive.metastore.DummyJdoConnectionUrlHook"); + hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, + DummyJdoConnectionUrlHook.initialUrl); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, + "org.apache.hadoop.hive.metastore.DummyRawStoreForJdoConnection"); + hiveConf.setBoolean("hive.metastore.checkForDefaultDb", true); + SessionState.start(new CliSessionState(hiveConf)); + + // Instantiating the HMSHandler with hive.metastore.checkForDefaultDb will cause it to + // initialize an instance of the DummyRawStoreForJdoConnection + HiveMetaStore.HMSHandler hms = new HiveMetaStore.HMSHandler( + "test_metastore_connection_url_hook_hms_handler", hiveConf); + } +} Index: metastore/src/test/org/apache/hadoop/hive/metastore/DummyJdoConnectionUrlHook.java =================================================================== --- /dev/null +++ metastore/src/test/org/apache/hadoop/hive/metastore/DummyJdoConnectionUrlHook.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; + +/** + * + * DummyJdoConnectionUrlHook. + * + * An implementation of JDOConnectionURLHook which simply returns CORRECT_URL when + * getJdoConnectionUrl is called. + */ +public class DummyJdoConnectionUrlHook implements JDOConnectionURLHook { + + public static final String initialUrl = "BAD_URL"; + public static final String newUrl = "CORRECT_URL"; + + @Override + public String getJdoConnectionUrl(Configuration conf) throws Exception { + return newUrl; + } + + @Override + public void notifyBadConnectionUrl(String url) { + } + +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -241,7 +241,7 @@ "hive.metastore.checkForDefaultDb", true); String alterHandlerName = hiveConf.get("hive.metastore.alter.impl", HiveAlterHandler.class.getName()); - alterHandler = (AlterHandler) ReflectionUtils.newInstance(getClass( + alterHandler = (AlterHandler) ReflectionUtils.newInstance(MetaStoreUtils.getClass( alterHandlerName, AlterHandler.class), hiveConf); wh = new Warehouse(hiveConf); @@ -310,10 +310,8 @@ LOG.info(addPrefix("Opening raw store with implemenation class:" + rawStoreClassName)); Configuration conf = getConf(); - RawStore ms = (RawStore) ReflectionUtils.newInstance(getClass(rawStoreClassName, - RawStore.class), conf); - return RetryingRawStore.getProxy(hiveConf, conf, ms, threadLocalId.get()); + return RetryingRawStore.getProxy(hiveConf, conf, rawStoreClassName, threadLocalId.get()); } private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException { @@ -350,15 +348,6 @@ } - private Class getClass(String rawStoreClassName, Class class1) - throws MetaException { - try { - return Class.forName(rawStoreClassName, true, classLoader); - } catch (ClassNotFoundException e) { - throw new MetaException(rawStoreClassName + " class not found"); - } - } - private void logInfo(String m) { LOG.info(threadLocalId.get().toString() + ": " + m); logAuditEvent(m); Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1015,4 +1015,13 @@ sd.setPrimaryRegionName(defaultRegionName); } } + + public static Class getClass(String rawStoreClassName, Class class1) + throws MetaException { + try { + return Class.forName(rawStoreClassName, true, JavaUtils.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new MetaException(rawStoreClassName + " class not found"); + } + } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java @@ -50,33 +50,39 @@ private final HiveConf hiveConf; private final Configuration conf; // thread local conf from HMS - protected RetryingRawStore(HiveConf hiveConf, Configuration conf, RawStore base, int id) - throws MetaException { - this.base = base; + protected RetryingRawStore(HiveConf hiveConf, Configuration conf, + Class rawStoreClass, int id) throws MetaException { this.conf = conf; this.hiveConf = hiveConf; this.id = id; + + // This has to be called before initializing the instance of RawStore init(); + + this.base = (RawStore) ReflectionUtils.newInstance(rawStoreClass, conf); } - public static RawStore getProxy(HiveConf hiveConf, Configuration conf, RawStore base, int id) - throws MetaException { + public static RawStore getProxy(HiveConf hiveConf, Configuration conf, String rawStoreClassName, + int id) throws MetaException { + + Class baseClass = (Class) MetaStoreUtils.getClass( + rawStoreClassName, RawStore.class); - RetryingRawStore handler = new RetryingRawStore(hiveConf, conf, base, id); + RetryingRawStore handler = new RetryingRawStore(hiveConf, conf, baseClass, id); return (RawStore) Proxy.newProxyInstance(RetryingRawStore.class.getClassLoader() - , base.getClass().getInterfaces(), handler); + , baseClass.getInterfaces(), handler); } private void init() throws MetaException { retryInterval = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.METASTOREINTERVAL); retryLimit = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.METASTOREATTEMPTS); // Using the hook on startup ensures that the hook always has priority - // over settings in *.xml. We can use hiveConf as only a single thread - // will be calling the constructor. - updateConnectionURL(hiveConf, null); + // over settings in *.xml. The thread local conf needs to be used because at this point + // it has already been initialized using hiveConf. + updateConnectionURL(getConf(), null); } private void initMS() {