diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 43c412d..d96d524 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -23,17 +23,13 @@ import java.sql.Connection; import java.sql.SQLException; -import java.sql.Statement; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -47,10 +43,13 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -65,9 +64,7 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; -import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.serde.serdeConstants; -import org.datanucleus.store.schema.SchemaTool; import com.google.common.collect.Lists; @@ -76,7 +73,7 @@ * the underlying database. It should use ANSI SQL and be compatible with common databases * such as MySQL (note that MySQL doesn't use full ANSI mode by default), Postgres, etc. * - * As of now, only the partition retrieval is done this way to improve job startup time; + * As of now, partition retrieval is done this way to improve job startup time; * JDOQL partition retrieval is still present so as not to limit the ORM solution we have * to SQL stores only. There's always a way to do without direct SQL. */ @@ -146,13 +143,19 @@ public boolean isCompatibleDatastore() { } /** - * See {@link #trySetAnsiQuotesForMysql()}. + * This function is intended to be called by functions before they put together a query + * Thus, any query-specific instantiation to be done from within the transaction is done + * here - for eg., for MySQL, we signal that we want to use ANSI SQL quoting behaviour */ - private void setAnsiQuotesForMysql() throws MetaException { - try { - trySetAnsiQuotesForMysql(); - } catch (SQLException sqlEx) { - throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + private void doDbSpecificInitializationsBeforeQuery() throws MetaException { + + if (isMySql) { + try { + assert pm.currentTransaction().isActive(); // must be inside tx together with queries + trySetAnsiQuotesForMysql(); + } catch (SQLException sqlEx) { + throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + } } } @@ -174,6 +177,79 @@ private void trySetAnsiQuotesForMysql() throws SQLException { } } + public Database getDatabase(String dbName) throws MetaException, NoSuchObjectException { + Query queryDbSelector = null; + Query queryDbParams = null; + try { + dbName = dbName.toLowerCase(); + + doDbSpecificInitializationsBeforeQuery(); + + String queryTextDbSelector= "select " + + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " + + "\"OWNER_NAME\", \"OWNER_TYPE\" " + + "FROM \"DBS\" where \"NAME\" = ? "; + Object[] params = new Object[] { dbName }; + queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector); + + LOG.debug("getDatabase:query instantiated : " + queryTextDbSelector + " with param ["+params[0]+"]"); + + List sqlResult = (List)queryDbSelector.executeWithArray(params); + if ((sqlResult == null) || sqlResult.isEmpty()) { + LOG.debug("getDatabase:queryDbSelector ran, returned no/empty results, returning NoSuchObjectException"); + throw new NoSuchObjectException("There is no database named " + dbName); + } + + assert(sqlResult.size() == 1); + if (sqlResult.get(0) == null){ + LOG.debug("getDatabase:queryDbSelector ran, returned results, but the result entry was null, returning NoSuchObjectException"); + throw new NoSuchObjectException("There is no database named " + dbName); + } + + Object[] dbline = sqlResult.get(0); + Long dbid = extractSqlLong(dbline[0]); + + String queryTextDbParams = "select \"PARAM_KEY\", \"PARAM_VALUE\" " + + " FROM \"DATABASE_PARAMS\" " + + " WHERE \"DB_ID\" = ? " + + " AND \"PARAM_KEY\" IS NOT NULL"; + Object[] params2 = new Object[] { dbid }; + queryDbParams = pm.newQuery("javax.jdo.query.SQL",queryTextDbParams); + LOG.debug("getDatabase:query2 instantiated : " + queryTextDbParams + " with param ["+params2[0]+"]"); + + Map dbParams = new HashMap(); + List sqlResult2 = ensureList(queryDbParams.executeWithArray(params2)); + if (!sqlResult2.isEmpty()){ + for (Object[] line : sqlResult2){ + dbParams.put(extractSqlString(line[0]),extractSqlString(line[1])); + } + } + + LOG.debug("getDatabase: instantiating db object to return"); + Database db = new Database(); + db.setName(extractSqlString(dbline[1])); + db.setLocationUri(extractSqlString(dbline[2])); + db.setDescription(extractSqlString(dbline[3])); + db.setOwnerName(extractSqlString(dbline[4])); + String type = extractSqlString(dbline[5]); + db.setOwnerType((null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); + db.setParameters(dbParams); + if (LOG.isDebugEnabled()){ + LOG.debug("getDatabase: directsql returning db " + db.getName() + + " locn["+db.getLocationUri() +"] desc [" +db.getDescription() + + "] owner [" + db.getOwnerName() + "] ownertype ["+ db.getOwnerType() +"]"); + } + return db; + } finally { + if (queryDbSelector != null){ + queryDbSelector.closeAll(); + } + if (queryDbParams != null){ + queryDbParams.closeAll(); + } + } + } + /** * Gets partitions by using direct SQL queries. * @param dbName Metastore db name. @@ -263,10 +339,8 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException tblName = tblName.toLowerCase(); // We have to be mindful of order during filtering if we are not returning all partitions. String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; - if (isMySql) { - assert pm.currentTransaction().isActive(); - setAnsiQuotesForMysql(); // must be inside tx together with queries - } + + doDbSpecificInitializationsBeforeQuery(); // Get all simple fields for partitions and related objects, which we can map one-on-one. // We will do this in 2 queries to use different existing indices for each one. @@ -599,6 +673,11 @@ public void apply(SerDeInfo t, Object[] fields) { return orderedResult; } + private String extractSqlString(Object value) { + if (value == null) return null; + return value.toString(); + } + private Long extractSqlLong(Object obj) throws MetaException { if (obj == null) return null; if (!(obj instanceof Number)) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 760777a..a8d153e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -491,14 +491,50 @@ private MDatabase getMDatabase(String name) throws NoSuchObjectException { @Override public Database getDatabase(String name) throws NoSuchObjectException { + try { + if (isDirectSqlEnabled(true) && directSql.isCompatibleDatastore()){ + Database db = null; + boolean committed = false; + try { + openTransaction(); + db = directSql.getDatabase(name); + committed = commitTransaction(); + return db; + } catch (JDODataStoreException jdoe) { + LOG.warn("JDODataStoreException using direct sql getting db: " + name + ". Falling back to ORM.",jdoe); + } finally { + if (!committed) { + rollbackTransaction(); + } + } + } + // If we reached here, then DirectSql was not enabled, or did not work. Fall back to JDO. + return getJDODatabase(name); + } catch (MetaException me){ + throw new NoSuchObjectException(me.getMessage()); + } + } + + public Database getJDODatabase(String name) throws NoSuchObjectException { MDatabase mdb = null; - boolean commited = false; + boolean committed = false; try { openTransaction(); mdb = getMDatabase(name); - commited = commitTransaction(); + committed = commitTransaction(); + } catch (Throwable t){ + if (t instanceof NoSuchObjectException){ + throw (NoSuchObjectException)t; + } else if (t instanceof RuntimeException){ + throw (RuntimeException)t; + } else if (t instanceof Error){ + throw (Error)t; + } else { + // We should not be throwing any checked exception that is not a NoSuchObjectException + throw new RuntimeException(t); + } } finally { - if (!commited) { + if (!committed) { rollbackTransaction(); } } @@ -576,7 +612,7 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc @Override public List getDatabases(String pattern) throws MetaException { - boolean commited = false; + boolean committed = false; List databases = null; try { openTransaction(); @@ -603,9 +639,9 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc for (Iterator i = names.iterator(); i.hasNext();) { databases.add((String) i.next()); } - commited = commitTransaction(); + committed = commitTransaction(); } finally { - if (!commited) { + if (!committed) { rollbackTransaction(); } } @@ -2165,6 +2201,15 @@ private void dropPartitionsNoTxn(String dbName, String tblName, List par return getPartitionsByFilterInternal(dbName, tblName, filter, maxParts, true, true); } + private boolean isDirectSqlEnabled(boolean allowSql) { + // We don't allow direct SQL usage if we are inside a larger transaction (e.g. droptable). + // That is because some databases (e.g. Postgres) abort the entire transaction when + // any query fails, so the fallback from failed SQL to JDO is not possible. + return allowSql + && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL) + && (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL_DDL) || !isActiveTransaction()); + } + /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */ private abstract class GetHelper { private final boolean isInTxn, doTrace, allowJdo; @@ -2184,11 +2229,7 @@ public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJ this.doTrace = LOG.isDebugEnabled(); this.isInTxn = isActiveTransaction(); - // SQL usage inside a larger transaction (e.g. droptable) may not be desirable because - // some databases (e.g. Postgres) abort the entire transaction when any query fails, so - // the fallback from failed SQL to JDO is not possible. - boolean isConfigEnabled = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL) - && (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL_DDL) || !isInTxn); + boolean isConfigEnabled = isDirectSqlEnabled(allowSql); if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) { throw new MetaException("SQL is not operational"); // test path; SQL is enabled and broken. }