Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreCommand.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreCommand.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreCommand.java (revision 0) @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * A MetaStoreCommand is a closure used to pass a block of code from individual + * functions to executeWithRetry, which centralizes connection error + * handling. MetaStoreCommand is parameterized on the return type of the function. + * + * The general transformation is: + * + * From: + * String foo(int a) throws ExceptionB { + * + * } + * + * To: + * String foo(final int a) throws ExceptionB { + * String ret = null; + * try { + * ret = executeWithRetry(new MetaStoreCommand() { + * String run(RawStore ms) { + * + * } + * } + * } catch (ExceptionB e) { + * throw e; + * } catch (Exception e) { + * // Since run is only supposed to throw ExceptionB it could only + * // be a runtime exception + * throw (RuntimeException)e; + * } + * } + * + * The catch blocks are used to ensure that the exceptions thrown by the + * follow the function definition. + */ +public abstract class MetaStoreCommand { + + private final ThreadLocal threadLocalMS; + private static final Log LOG = LogFactory.getLog(MetaStoreCommand.class); + + public MetaStoreCommand(ThreadLocal threadLocalMS) { + super(); + this.threadLocalMS = threadLocalMS; + } + + public abstract T run(RawStore ms) throws Exception; + + public T executeWithRetry(URLConnectionUpdater updater, HiveConf conf) throws Exception { + T ret = null; + + boolean gotNewConnectUrl = false; + boolean reloadConf = conf.getBoolVar(HiveConf.ConfVars.METASTOREFORCERELOADCONF); + // Used for retrying JDO calls on datastore failures + int retryInterval = HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTOREINTERVAL); + int retryLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTOREATTEMPTS); + if (reloadConf) { + updater.updateConnectionURL(conf, null); + } + + int retryCount = 0; + Exception caughtException = null; + while(true) { + try { + RawStore ms = getMS(conf, reloadConf || gotNewConnectUrl); + ret = run(ms); + break; + } catch (javax.jdo.JDOException e) { + caughtException = e; + } + + if (retryCount >= retryLimit) { + throw caughtException; + } + + assert(retryInterval >= 0); + retryCount++; + LOG.error(String.format( "JDO datastore error. Retrying metastore command " + + "after %d ms (attempt %d of %d)", retryInterval, retryCount, retryLimit)); + Thread.sleep(retryInterval); + // If we have a connection error, the JDO connection URL hook might + // provide us with a new URL to access the datastore. + String lastUrl = URLConnectionUpdater.getConnectionURL(conf); + gotNewConnectUrl = updater.updateConnectionURL(conf, lastUrl); + } + return ret; + } + + /** + * Get a cached RawStore. + * + * @return + * @throws MetaException + */ + private RawStore getMS(HiveConf conf, boolean reloadConf) throws MetaException { + RawStore ms = threadLocalMS.get(); + String rawStoreClassName = conf.get("hive.metastore.rawstore.impl"); + if (ms == null) { + LOG.info(MetaStoreUtils.addPrefix("Opening raw store with implemenation class:" + + rawStoreClassName)); + ms = (RawStore) ReflectionUtils.newInstance(MetaStoreUtils.getClass(rawStoreClassName, + RawStore.class), conf); + threadLocalMS.set(ms); + ms = threadLocalMS.get(); + } + + if (reloadConf) { + ms.setConf(conf); + } + + return ms; + } +} \ No newline at end of file Index: metastore/src/java/org/apache/hadoop/hive/metastore/URLConnectionUpdater.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/URLConnectionUpdater.java (revision 0) +++ metastore/src/java/org/apache/hadoop/hive/metastore/URLConnectionUpdater.java (revision 0) @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; +import org.apache.hadoop.util.ReflectionUtils; + +public class URLConnectionUpdater { + + private final Configuration config; + private JDOConnectionURLHook urlHook = null; + private String urlHookClassName = ""; + private static final Log LOG = LogFactory.getLog(URLConnectionUpdater.class); + + public URLConnectionUpdater(Configuration config) throws MetaException { + super(); + this.config = config; + updateConnectionURL(config, null); + } + + /** + * Updates the connection URL in hiveConf using the hook + * @return true if a new connection URL was loaded into the thread local + * configuration + */ + boolean updateConnectionURL(Configuration conf, String badUrl) + throws MetaException { + String connectUrl = null; + String currentUrl = getConnectionURL(conf); + try { + // We always call init because the hook name in the configuration could + // have changed. + initConnectionUrlHook(); + if (urlHook != null) { + if (badUrl != null) { + urlHook.notifyBadConnectionUrl(badUrl); + } + connectUrl = urlHook.getJdoConnectionUrl(config); + } + } catch (Exception e) { + LOG.error("Exception while getting connection URL from the hook: " + + e); + } + + if (connectUrl != null && !connectUrl.equals(currentUrl)) { + LOG.error(MetaStoreUtils.addPrefix( + String.format("Overriding %s with %s", + HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), + connectUrl))); + conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), + connectUrl); + return true; + } + return false; + } + + static String getConnectionURL(Configuration conf) { + return conf.get( + HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), ""); + } + + // Multiple threads could try to initialize at the same time. + synchronized private void initConnectionUrlHook() + throws ClassNotFoundException { + + String className = + config.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim(); + if (className.equals("")) { + urlHookClassName = ""; + urlHook = null; + return; + } + boolean urlHookChanged = !urlHookClassName.equals(className); + if (urlHook == null || urlHookChanged) { + urlHookClassName = className.trim(); + + Class urlHookClass = Class.forName(urlHookClassName, true, + JavaUtils.getClassLoader()); + urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null); + } + return; + } +} Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1096976) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -37,7 +37,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.metrics.Metrics; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -65,7 +64,6 @@ import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; -import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; @@ -108,7 +106,6 @@ ThriftHiveMetastore.Iface { public static final Log LOG = HiveMetaStore.LOG; private static boolean createDefaultDB = false; - private String rawStoreClassName; private final HiveConf hiveConf; // stores datastore (jpox) properties, // right now they come from jpox.properties @@ -166,18 +163,14 @@ // The next serial number to be assigned private boolean checkForDefaultDb; private static int nextSerialNum = 0; - private static ThreadLocal threadLocalId = new ThreadLocal() { + private static ThreadLocal threadLocalId = new ThreadLocal() { @Override - protected synchronized Object initialValue() { + protected synchronized Integer initialValue() { return new Integer(nextSerialNum++); } }; - // Used for retrying JDO calls on datastore failures - private int retryInterval = 0; - private int retryLimit = 0; - private JDOConnectionURLHook urlHook = null; - private String urlHookClassName = ""; + private URLConnectionUpdater connUpdater; public static Integer get() { return threadLocalId.get(); @@ -199,33 +192,21 @@ return hiveConf; } - private ClassLoader classLoader; private AlterHandler alterHandler; - { - classLoader = Thread.currentThread().getContextClassLoader(); - if (classLoader == null) { - classLoader = Configuration.class.getClassLoader(); - } - } private boolean init() throws MetaException { - rawStoreClassName = hiveConf.get("hive.metastore.rawstore.impl"); checkForDefaultDb = hiveConf.getBoolean( "hive.metastore.checkForDefaultDb", true); String alterHandlerName = hiveConf.get("hive.metastore.alter.impl", HiveAlterHandler.class.getName()); - alterHandler = (AlterHandler) ReflectionUtils.newInstance(getClass( + alterHandler = (AlterHandler) ReflectionUtils.newInstance(MetaStoreUtils.getClass( alterHandlerName, AlterHandler.class), hiveConf); wh = new Warehouse(hiveConf); - retryInterval = HiveConf.getIntVar(hiveConf, - HiveConf.ConfVars.METASTOREINTERVAL); - retryLimit = HiveConf.getIntVar(hiveConf, - HiveConf.ConfVars.METASTOREATTEMPTS); // Using the hook on startup ensures that the hook always has priority // over settings in *.xml. We can use hiveConf as only a single thread // will be calling the constructor. - updateConnectionURL(hiveConf, null); + connUpdater = new URLConnectionUpdater(hiveConf); createDefaultDB(); @@ -243,90 +224,6 @@ return true; } - private String addPrefix(String s) { - return threadLocalId.get() + ": " + s; - } - - /** - * A Command is a closure used to pass a block of code from individual - * functions to executeWithRetry, which centralizes connection error - * handling. Command is parameterized on the return type of the function. - * - * The general transformation is: - * - * From: - * String foo(int a) throws ExceptionB { - * - * } - * - * To: - * String foo(final int a) throws ExceptionB { - * String ret = null; - * try { - * ret = executeWithRetry(new Command() { - * String run(RawStore ms) { - * - * } - * } - * } catch (ExceptionB e) { - * throw e; - * } catch (Exception e) { - * // Since run is only supposed to throw ExceptionB it could only - * // be a runtime exception - * throw (RuntimeException)e; - * } - * } - * - * The catch blocks are used to ensure that the exceptions thrown by the - * follow the function definition. - */ - private static class Command { - T run(RawStore ms) throws Exception { - return null; - } - } - - private T executeWithRetry(Command cmd) throws Exception { - T ret = null; - - boolean gotNewConnectUrl = false; - boolean reloadConf = HiveConf.getBoolVar(hiveConf, - HiveConf.ConfVars.METASTOREFORCERELOADCONF); - - if (reloadConf) { - updateConnectionURL(getConf(), null); - } - - int retryCount = 0; - Exception caughtException = null; - while(true) { - try { - RawStore ms = getMS(reloadConf || gotNewConnectUrl); - ret = cmd.run(ms); - break; - } catch (javax.jdo.JDOException e) { - caughtException = e; - } - - if (retryCount >= retryLimit) { - throw caughtException; - } - - assert(retryInterval >= 0); - retryCount++; - LOG.error( - String.format( - "JDO datastore error. Retrying metastore command " + - "after %d ms (attempt %d of %d)", retryInterval, retryCount, retryLimit)); - Thread.sleep(retryInterval); - // If we have a connection error, the JDO connection URL hook might - // provide us with a new URL to access the datastore. - String lastUrl = getConnectionURL(getConf()); - gotNewConnectUrl = updateConnectionURL(getConf(), lastUrl); - } - return ret; - } - private Configuration getConf() { Configuration conf = threadLocalConf.get(); if (conf == null) { @@ -336,93 +233,6 @@ return conf; } - /** - * Get a cached RawStore. - * - * @return - * @throws MetaException - */ - private RawStore getMS(boolean reloadConf) throws MetaException { - RawStore ms = threadLocalMS.get(); - if (ms == null) { - LOG.info(addPrefix("Opening raw store with implemenation class:" - + rawStoreClassName)); - ms = (RawStore) ReflectionUtils.newInstance(getClass(rawStoreClassName, - RawStore.class), getConf()); - threadLocalMS.set(ms); - ms = threadLocalMS.get(); - } - - if (reloadConf) { - ms.setConf(getConf()); - } - - return ms; - } - - /** - * Updates the connection URL in hiveConf using the hook - * @return true if a new connection URL was loaded into the thread local - * configuration - */ - private boolean updateConnectionURL(Configuration conf, String badUrl) - throws MetaException { - String connectUrl = null; - String currentUrl = getConnectionURL(conf); - try { - // We always call init because the hook name in the configuration could - // have changed. - initConnectionUrlHook(); - if (urlHook != null) { - if (badUrl != null) { - urlHook.notifyBadConnectionUrl(badUrl); - } - connectUrl = urlHook.getJdoConnectionUrl(hiveConf); - } - } catch (Exception e) { - LOG.error("Exception while getting connection URL from the hook: " + - e); - } - - if (connectUrl != null && !connectUrl.equals(currentUrl)) { - LOG.error(addPrefix( - String.format("Overriding %s with %s", - HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), - connectUrl))); - conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), - connectUrl); - return true; - } - return false; - } - - private static String getConnectionURL(Configuration conf) { - return conf.get( - HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), ""); - } - - // Multiple threads could try to initialize at the same time. - synchronized private void initConnectionUrlHook() - throws ClassNotFoundException { - - String className = - hiveConf.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim(); - if (className.equals("")) { - urlHookClassName = ""; - urlHook = null; - return; - } - boolean urlHookChanged = !urlHookClassName.equals(className); - if (urlHook == null || urlHookChanged) { - urlHookClassName = className.trim(); - - Class urlHookClass = Class.forName(urlHookClassName, true, - JavaUtils.getClassLoader()); - urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null); - } - return; - } - private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException { try { ms.getDatabase(DEFAULT_DATABASE_NAME); @@ -444,13 +254,13 @@ } try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { createDefaultDB_core(ms); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidObjectException e) { throw new MetaException(e.getMessage()); } catch (MetaException e) { @@ -462,15 +272,6 @@ } - private Class getClass(String rawStoreClassName, Class class1) - throws MetaException { - try { - return Class.forName(rawStoreClassName, true, classLoader); - } catch (ClassNotFoundException e) { - throw new MetaException(rawStoreClassName + " class not found"); - } - } - private void logInfo(String m) { LOG.info(threadLocalId.get().toString() + ": " + m); logAuditEvent(m); @@ -563,13 +364,13 @@ } catch (NoSuchObjectException e) { // expected } - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { create_database_core(ms, db); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (AlreadyExistsException e) { throw e; } catch (InvalidObjectException e) { @@ -589,12 +390,12 @@ startFunction("get_database", ": " + name); Database db = null; try { - db = executeWithRetry(new Command() { + db = new MetaStoreCommand(threadLocalMS) { @Override - Database run(RawStore ms) throws Exception { + public Database run(RawStore ms) throws Exception { return ms.getDatabase(name); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -612,12 +413,12 @@ throws NoSuchObjectException, TException, MetaException { startFunction("alter_database" + dbName); try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return ms.alterDatabase(dbName, db); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -674,13 +475,13 @@ } try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { drop_database_core(ms, dbName, deleteData, cascade); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (InvalidOperationException e) { @@ -700,12 +501,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getDatabases(pattern); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -722,12 +523,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getAllDatabases(); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -765,13 +566,13 @@ startFunction("create_type", ": " + type.getName()); Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { create_type_core(ms, type); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (AlreadyExistsException e) { throw e; } catch (MetaException e) { @@ -793,16 +594,16 @@ Type ret; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Type run(RawStore ms) throws Exception { + public Type run(RawStore ms) throws Exception { Type type = ms.getType(name); if (null == type) { throw new NoSuchObjectException("Type \"" + name + "\" not found."); } return type; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (MetaException e) { @@ -826,7 +627,6 @@ boolean success = false; try { ms.openTransaction(); - // drop any partitions if (!is_type_exists(ms, typeName)) { throw new NoSuchObjectException(typeName + " doesn't exist"); } @@ -847,13 +647,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { // TODO:pc validate that there are no types that refer to this return ms.dropType(name); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -944,13 +744,13 @@ startFunction("create_table", ": db=" + tbl.getDbName() + " tbl=" + tbl.getTableName()); try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { create_table_core(ms, tbl); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (AlreadyExistsException e) { throw e; } catch (MetaException e) { @@ -1040,13 +840,13 @@ startTableFunction("drop_table", dbname, name); try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { drop_table_core(ms, dbname, name, deleteData); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (MetaException e) { @@ -1080,9 +880,9 @@ Table t = null; startTableFunction("get_table", dbname, name); try { - t = executeWithRetry(new Command() { + t = new MetaStoreCommand
(threadLocalMS) { @Override - Table run(RawStore ms) throws Exception { + public Table run(RawStore ms) throws Exception { Table t = ms.getTable(dbname, name); if (t == null) { throw new NoSuchObjectException(dbname + "." + name @@ -1090,7 +890,7 @@ } return t; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (MetaException e) { @@ -1192,12 +992,12 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { return append_partition_common(ms, dbName, tableName, part_vals); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (InvalidObjectException e) { @@ -1244,13 +1044,13 @@ Integer ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Integer run(RawStore ms) throws Exception { + public Integer run(RawStore ms) throws Exception { int ret = add_partitions_core(ms, parts); return Integer.valueOf(ret); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidObjectException e) { throw e; } catch (AlreadyExistsException e) { @@ -1351,12 +1151,12 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { return add_partition_core(ms, part); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidObjectException e) { throw e; } catch (AlreadyExistsException e) { @@ -1443,13 +1243,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return Boolean.valueOf( drop_partition_common(ms, db_name, tbl_name, part_vals, deleteData)); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1472,12 +1272,12 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { return ms.getPartition(db_name, tbl_name, part_vals); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1501,13 +1301,13 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { return ms.getPartitionWithAuth(db_name, tbl_name, part_vals, user_name, group_names); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1527,12 +1327,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getPartitions(db_name, tbl_name, max_parts); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1556,13 +1356,13 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getPartitionsWithAuth(dbName, tblName, maxParts, userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1583,12 +1383,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.listPartitionNames(db_name, tbl_name, max_parts); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -1624,13 +1424,13 @@ LOG.info("Partition values:" + new_part.getValues()); try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { alter_partition_core(ms, db_name, tbl_name, new_part); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidOperationException e) { throw e; } catch (MetaException e) { @@ -1662,13 +1462,13 @@ .currentTimeMillis() / 1000)); try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { ms.alterIndex(dbname, base_table_name, index_name, newIndex); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (InvalidOperationException e) { @@ -1701,13 +1501,13 @@ try { - executeWithRetry(new Command() { + new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { alterHandler.alterTable(ms, wh, dbname, name, newTable); return Boolean.TRUE; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (InvalidOperationException e) { @@ -1726,12 +1526,12 @@ List ret; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getTables(dbname, pattern); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -1748,12 +1548,12 @@ List ret; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getAllTables(dbname); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -1930,12 +1730,12 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { return get_partition_by_name_core(ms, db_name, tbl_name, part_name); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -1959,13 +1759,13 @@ Partition ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Partition run(RawStore ms) throws Exception { + public Partition run(RawStore ms) throws Exception { List partVals = getPartValsFromName(ms, db_name, tbl_name, part_name); return append_partition_common(ms, db_name, tbl_name, partVals); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidObjectException e) { throw e; } catch (AlreadyExistsException e) { @@ -2007,13 +1807,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return drop_partition_by_name_core(ms, db_name, tbl_name, part_name, deleteData); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (MetaException e) { @@ -2085,9 +1885,8 @@ throws MetaException, TException { startPartitionFunction("get_partitions_names_ps", db_name, tbl_name, part_vals); try { - Table t; try { - t = get_table(db_name, tbl_name); + get_table(db_name, tbl_name); } catch (NoSuchObjectException e) { throw new MetaException(e.getMessage()); } @@ -2142,12 +1941,12 @@ + newIndex.getOrigTableName() + " index=" + newIndex.getIndexName()); Index ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Index run(RawStore ms) throws Exception { + public Index run(RawStore ms) throws Exception { return add_index_core(ms, newIndex, indexTable); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (InvalidObjectException e) { throw e; } catch (AlreadyExistsException e) { @@ -2229,13 +2028,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return drop_index_by_name_core(ms, dbName, tblName, indexName, deleteData); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (NoSuchObjectException e) { throw e; } catch (MetaException e) { @@ -2314,12 +2113,12 @@ Index ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Index run(RawStore ms) throws Exception { + public Index run(RawStore ms) throws Exception { return get_index_by_name_core(ms, dbName, tblName, indexName); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -2354,12 +2153,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.listIndexNames(dbName, tblName, maxIndexes); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2379,12 +2178,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getIndexes(dbName, tblName, maxIndexes); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2404,12 +2203,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getPartitionsByFilter(dbName, tblName, filter, maxParts); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -2432,12 +2231,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.getPartitionsByNames(dbName, tblName, partNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (NoSuchObjectException e) { @@ -2498,13 +2297,13 @@ PrincipalPrivilegeSet ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - PrincipalPrivilegeSet run(RawStore ms) throws Exception { + public PrincipalPrivilegeSet run(RawStore ms) throws Exception { return ms.getColumnPrivilegeSet( dbName, tableName, partName, columnName, userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2520,12 +2319,12 @@ PrincipalPrivilegeSet ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - PrincipalPrivilegeSet run(RawStore ms) throws Exception { + public PrincipalPrivilegeSet run(RawStore ms) throws Exception { return ms.getDBPrivilegeSet(dbName, userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2542,13 +2341,13 @@ PrincipalPrivilegeSet ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - PrincipalPrivilegeSet run(RawStore ms) throws Exception { + public PrincipalPrivilegeSet run(RawStore ms) throws Exception { return ms.getPartitionPrivilegeSet(dbName, tableName, partName, userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2564,13 +2363,13 @@ PrincipalPrivilegeSet ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - PrincipalPrivilegeSet run(RawStore ms) throws Exception { + public PrincipalPrivilegeSet run(RawStore ms) throws Exception { return ms.getTablePrivilegeSet(dbName, tableName, userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2588,13 +2387,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { Role role = ms.getRole(roleName); return ms.grantRole(role, userName, principalType, grantor, grantorType, grantOption); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2609,9 +2408,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { List result = new ArrayList(); List roleMap = ms.listRoles(principalName, principalType); if (roleMap!=null) { @@ -2623,7 +2422,7 @@ } return result; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2640,12 +2439,12 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return ms.addRole(role.getRoleName(), role.getOwnerName()); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2661,12 +2460,12 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return ms.removeRole(roleName); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2681,12 +2480,12 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { return ms.listRoleNames(); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2703,12 +2502,12 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return ms.grantPrivileges(privileges); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { e.printStackTrace(); throw e; @@ -2725,13 +2524,13 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { Role mRole = ms.getRole(roleName); return ms.revokeRole(mRole, userName, principalType); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2747,12 +2546,12 @@ Boolean ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - Boolean run(RawStore ms) throws Exception { + public Boolean run(RawStore ms) throws Exception { return ms.revokePrivileges(privileges); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2767,12 +2566,12 @@ PrincipalPrivilegeSet ret = null; try { - ret = executeWithRetry(new Command() { + ret = new MetaStoreCommand(threadLocalMS) { @Override - PrincipalPrivilegeSet run(RawStore ms) throws Exception { + public PrincipalPrivilegeSet run(RawStore ms) throws Exception { return ms.getUserPrivilegeSet(userName, groupNames); } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2817,9 +2616,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { String partName = null; if (partValues != null && partValues.size()>0) { Table tbl = get_table(dbName, tableName); @@ -2874,7 +2673,7 @@ return result; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2890,9 +2689,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { List mDbs = ms.listPrincipalDBGrants( principalName, principalType, dbName); if (mDbs.size() > 0) { @@ -2912,7 +2711,7 @@ } return null; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2929,9 +2728,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { Table tbl = get_table(dbName, tableName); String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues); List mParts = ms.listPrincipalPartitionGrants( @@ -2956,7 +2755,7 @@ } return null; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -2973,9 +2772,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { List mTbls = ms .listAllTableGrants(principalName, principalType, dbName, tableName); if (mTbls.size() > 0) { @@ -2995,7 +2794,7 @@ } return null; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { @@ -3011,9 +2810,9 @@ List ret = null; try { - ret = executeWithRetry(new Command>() { + ret = new MetaStoreCommand>(threadLocalMS) { @Override - List run(RawStore ms) throws Exception { + public List run(RawStore ms) throws Exception { List mUsers = ms.listPrincipalGlobalGrants( principalName, principalType); if (mUsers.size() > 0) { @@ -3033,7 +2832,7 @@ } return null; } - }); + }.executeWithRetry(connUpdater,getHiveConf()); } catch (MetaException e) { throw e; } catch (Exception e) { Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1096976) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Constants; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -944,4 +945,18 @@ } return filter.toString(); } + + public static String addPrefix(String s) { + return HiveMetaStore.HMSHandler.get() + ": " + s; } + + public static Class getClass(String rawStoreClassName, Class class1) + throws MetaException { + try { + return Class.forName(rawStoreClassName, true, JavaUtils.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new MetaException(rawStoreClassName + " class not found"); + } + } + +}