diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 1c21c8b..5776ec6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -178,25 +178,44 @@ private String getProductName() { private boolean ensureDbInit() { Transaction tx = pm.currentTransaction(); + Query dbQuery = null, tblColumnQuery = null, partColumnQuery = null; try { // Force the underlying db to initialize. - pm.newQuery(MDatabase.class, "name == ''").execute(); - pm.newQuery(MTableColumnStatistics.class, "dbName == ''").execute(); - pm.newQuery(MPartitionColumnStatistics.class, "dbName == ''").execute(); + dbQuery = pm.newQuery(MDatabase.class, "name == ''"); + dbQuery.execute(); + + tblColumnQuery = pm.newQuery(MTableColumnStatistics.class, "dbName == ''"); + tblColumnQuery.execute(); + + partColumnQuery = pm.newQuery(MPartitionColumnStatistics.class, "dbName == ''"); + partColumnQuery.execute(); + return true; } catch (Exception ex) { LOG.warn("Database initialization failed; direct SQL is disabled", ex); tx.rollback(); return false; + } finally { + if (dbQuery != null) { + dbQuery.closeAll(); + } + if (tblColumnQuery != null) { + tblColumnQuery.closeAll(); + } + if (partColumnQuery != null) { + partColumnQuery.closeAll(); + } } } private boolean runTestQuery() { Transaction tx = pm.currentTransaction(); + Query query = null; // Run a self-test query. If it doesn't work, we will self-disable. What a PITA... String selfTestQuery = "select \"DB_ID\" from \"DBS\""; try { - pm.newQuery("javax.jdo.query.SQL", selfTestQuery).execute(); + query = pm.newQuery("javax.jdo.query.SQL", selfTestQuery); + query.execute(); tx.commit(); return true; } catch (Exception ex) { @@ -204,6 +223,11 @@ private boolean runTestQuery() { tx.rollback(); return false; } + finally { + if (query != null) { + query.closeAll(); + } + } } public boolean isCompatibleDatastore() { @@ -393,14 +417,21 @@ private static Boolean isViewTable(Table t) { } private boolean isViewTable(String dbName, String tblName) throws MetaException { - String queryText = "select \"TBL_TYPE\" from \"TBLS\"" + - " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " + - " where \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ?"; - Object[] params = new Object[] { tblName, dbName }; - Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - query.setUnique(true); - Object result = executeWithArray(query, params, queryText); - return (result != null) && result.toString().equals(TableType.VIRTUAL_VIEW.toString()); + Query query = null; + try { + String queryText = "select \"TBL_TYPE\" from \"TBLS\"" + + " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " + + " where \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ?"; + Object[] params = new Object[] { tblName, dbName }; + query = pm.newQuery("javax.jdo.query.SQL", queryText); + query.setUnique(true); + Object result = executeWithArray(query, params, queryText); + return (result != null) && result.toString().equals(TableType.VIRTUAL_VIEW.toString()); + } finally { + if (query != null) { + query.closeAll(); + } + } } /** @@ -1190,6 +1221,7 @@ private long partsFoundForPartitions(String dbName, String tableName, partsFound++; } } + query.closeAll(); return partsFound; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 8f52f83..39ab9e7 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -217,6 +217,29 @@ private Pattern partitionValidationPattern; + /** + * A class to pass the Query object to the caller to let the caller release + * resources by calling QueryWrapper.query.closeAll() after consuming all the query results. + */ + public static class QueryWrapper { + public Query query; + + /** + * Explicitly closes the query object to release the resources + */ + public void close() { + if (query != null) { + query.closeAll(); + query = null; + } + } + + @Override + protected void finalize() { + this.close(); + } + } + public ObjectStore() { } @@ -551,10 +574,11 @@ public void createDatabase(Database db) throws InvalidObjectException, MetaExcep private MDatabase getMDatabase(String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; + Query query = null; try { openTransaction(); name = HiveStringUtils.normalizeIdentifier(name); - Query query = pm.newQuery(MDatabase.class, "name == dbname"); + query = pm.newQuery(MDatabase.class, "name == dbname"); query.declareParameters("java.lang.String dbname"); query.setUnique(true); mdb = (MDatabase) query.execute(name); @@ -564,6 +588,9 @@ private MDatabase getMDatabase(String name) throws NoSuchObjectException { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } if (mdb == null) { throw new NoSuchObjectException("There is no database named " + name); @@ -666,6 +693,7 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc boolean success = false; LOG.info("Dropping database " + dbname + " along with all tables"); dbname = HiveStringUtils.normalizeIdentifier(dbname); + QueryWrapper queryWrapper = new QueryWrapper(); try { openTransaction(); @@ -673,7 +701,7 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc MDatabase db = getMDatabase(dbname); pm.retrieve(db); if (db != null) { - List dbGrants = this.listDatabaseGrants(dbname); + List dbGrants = this.listDatabaseGrants(dbname, queryWrapper); if (dbGrants != null && dbGrants.size() > 0) { pm.deletePersistentAll(dbGrants); } @@ -684,36 +712,36 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc if (!success) { rollbackTransaction(); } + queryWrapper.close(); } return success; } - @Override public List getDatabases(String pattern) throws MetaException { boolean commited = false; List databases = null; + Query query = null; try { openTransaction(); // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = "select name from org.apache.hadoop.hive.metastore.model.MDatabase where ("; + String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MDatabase where ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " name.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " name.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.setResult("name"); - q.setOrdering("name ascending"); - Collection names = (Collection) q.execute(); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.setResult("name"); + query.setOrdering("name ascending"); + Collection names = (Collection) query.execute(); databases = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { databases.add((String) i.next()); @@ -723,6 +751,9 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return databases; } @@ -781,9 +812,10 @@ public boolean createType(Type type) { public Type getType(String typeName) { Type type = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MType.class, "name == typeName"); + query = pm.newQuery(MType.class, "name == typeName"); query.declareParameters("java.lang.String typeName"); query.setUnique(true); MType mtype = (MType) query.execute(typeName.trim()); @@ -796,6 +828,9 @@ public Type getType(String typeName) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return type; } @@ -803,9 +838,10 @@ public Type getType(String typeName) { @Override public boolean dropType(String typeName) { boolean success = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MType.class, "name == typeName"); + query = pm.newQuery(MType.class, "name == typeName"); query.declareParameters("java.lang.String typeName"); query.setUnique(true); MType type = (MType) query.execute(typeName.trim()); @@ -821,6 +857,9 @@ public boolean dropType(String typeName) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return success; } @@ -956,9 +995,9 @@ public Table getTable(String dbName, String tableName) throws MetaException { } @Override - public List getTables(String dbName, String pattern) - throws MetaException { + public List getTables(String dbName, String pattern) throws MetaException { boolean commited = false; + Query query = null; List tbls = null; try { openTransaction(); @@ -966,25 +1005,24 @@ public Table getTable(String dbName, String tableName) throws MetaException { // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = - "select tableName from org.apache.hadoop.hive.metastore.model.MTable " - + "where database.name == dbName && ("; + String queryStr = + "select tableName from org.apache.hadoop.hive.metastore.model.MTable " + + "where database.name == dbName && ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " tableName.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " tableName.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.declareParameters("java.lang.String dbName"); - q.setResult("tableName"); - q.setOrdering("tableName ascending"); - Collection names = (Collection) q.execute(dbName); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.declareParameters("java.lang.String dbName"); + query.setResult("tableName"); + query.setOrdering("tableName ascending"); + Collection names = (Collection) query.execute(dbName); tbls = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { tbls.add((String) i.next()); @@ -994,6 +1032,9 @@ public Table getTable(String dbName, String tableName) throws MetaException { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return tbls; } @@ -1006,11 +1047,12 @@ public Table getTable(String dbName, String tableName) throws MetaException { private MTable getMTable(String db, String table) { MTable mtbl = null; boolean commited = false; + Query query = null; try { openTransaction(); db = HiveStringUtils.normalizeIdentifier(db); table = HiveStringUtils.normalizeIdentifier(table); - Query query = pm.newQuery(MTable.class, "tableName == table && database.name == db"); + query = pm.newQuery(MTable.class, "tableName == table && database.name == db"); query.declareParameters("java.lang.String table, java.lang.String db"); query.setUnique(true); mtbl = (MTable) query.execute(table, db); @@ -1020,20 +1062,24 @@ private MTable getMTable(String db, String table) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mtbl; } @Override - public List getTableObjectsByName(String db, List tbl_names) - throws MetaException, UnknownDBException { + public List
getTableObjectsByName(String db, List tbl_names) throws MetaException, + UnknownDBException { List
tables = new ArrayList
(); boolean committed = false; + Query dbExistsQuery = null; + Query query = null; try { openTransaction(); - db = HiveStringUtils.normalizeIdentifier(db); - Query dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); + dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); dbExistsQuery.declareParameters("java.lang.String db"); dbExistsQuery.setUnique(true); dbExistsQuery.setResult("name"); @@ -1046,7 +1092,7 @@ private MTable getMTable(String db, String table) { for (String t : tbl_names) { lowered_tbl_names.add(HiveStringUtils.normalizeIdentifier(t)); } - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); query.setFilter("database.name == db && tbl_names.contains(tableName)"); query.declareParameters("java.lang.String db, java.util.Collection tbl_names"); Collection mtables = (Collection) query.execute(db, lowered_tbl_names); @@ -1058,6 +1104,12 @@ private MTable getMTable(String db, String table) { if (!committed) { rollbackTransaction(); } + if (dbExistsQuery != null) { + dbExistsQuery.closeAll(); + } + if (query != null) { + query.closeAll(); + } } return tables; } @@ -1208,9 +1260,9 @@ private MColumnDescriptor createNewMColumnDescriptor(List cols) { // MSD and SD should be same objects. Not sure how to make then same right now // MSerdeInfo *& SerdeInfo should be same as well - private StorageDescriptor convertToStorageDescriptor(MStorageDescriptor msd, - boolean noFS) - throws MetaException { + private StorageDescriptor convertToStorageDescriptor( + MStorageDescriptor msd, + boolean noFS) throws MetaException { if (msd == null) { return null; } @@ -1296,8 +1348,6 @@ private StorageDescriptor convertToStorageDescriptor(MStorageDescriptor msd) return map; } - - /** * Converts a storage descriptor to a db-backed storage descriptor. Creates a * new db-backed column descriptor object for this SD. @@ -1404,7 +1454,6 @@ private boolean isValidPartition( return !doesExist; } - @Override public boolean addPartitions(String dbName, String tblName, PartitionSpecProxy partitionSpec, boolean ifNotExists) @@ -1531,10 +1580,11 @@ public Partition getPartition(String dbName, String tableName, return part; } - private MPartition getMPartition(String dbName, String tableName, - List part_vals) throws MetaException { + private MPartition getMPartition(String dbName, String tableName, List part_vals) + throws MetaException { MPartition mpart = null; boolean commited = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -1546,10 +1596,11 @@ private MPartition getMPartition(String dbName, String tableName, } // Change the query to use part_vals instead of the name which is // redundant TODO: callers of this often get part_vals out of name for no reason... - String name = Warehouse.makePartName(convertToFieldSchemas(mtbl - .getPartitionKeys()), part_vals); - Query query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2 && partitionName == t3"); + String name = + Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals); + query = + pm.newQuery(MPartition.class, + "table.tableName == t1 && table.database.name == t2 && partitionName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); mpart = (MPartition) query.execute(tableName, dbName, name); @@ -1559,6 +1610,9 @@ private MPartition getMPartition(String dbName, String tableName, if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mpart; } @@ -1750,8 +1804,13 @@ private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectExceptio } @Override protected List getJdoResult( - GetHelper> ctx) throws MetaException, NoSuchObjectException { - return convertToParts(listMPartitions(dbName, tblName, maxParts)); + GetHelper> ctx) throws MetaException { + QueryWrapper queryWrapper = new QueryWrapper(); + try { + return convertToParts(listMPartitions(dbName, tblName, maxParts, queryWrapper)); + } finally { + queryWrapper.close(); + } } }.run(false); } @@ -1759,11 +1818,13 @@ private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectExceptio @Override public List getPartitionsWithAuth(String dbName, String tblName, short max, String userName, List groupNames) - throws MetaException, NoSuchObjectException, InvalidObjectException { + throws MetaException, InvalidObjectException { boolean success = false; + QueryWrapper queryWrapper = new QueryWrapper(); + try { openTransaction(); - List mparts = listMPartitions(dbName, tblName, max); + List mparts = listMPartitions(dbName, tblName, max, queryWrapper); List parts = new ArrayList(mparts.size()); if (mparts != null && mparts.size()>0) { for (MPartition mpart : mparts) { @@ -1786,6 +1847,7 @@ private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectExceptio if (!success) { rollbackTransaction(); } + queryWrapper.close(); } } @@ -1822,7 +1884,6 @@ public Partition getPartitionWithAuth(String dbName, String tblName, } } - private List convertToParts(List mparts) throws MetaException { return convertToParts(mparts, null); } @@ -1875,20 +1936,22 @@ public Partition getPartitionWithAuth(String dbName, String tblName, List pns = new ArrayList(); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - Query q = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.database.name == t1 && table.tableName == t2 " - + "order by partitionName asc"); - q.declareParameters("java.lang.String t1, java.lang.String t2"); - q.setResult("partitionName"); - - if(max > 0) { - q.setRange(0, max); - } - Collection names = (Collection) q.execute(dbName, tableName); + Query query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where table.database.name == t1 && table.tableName == t2 " + + "order by partitionName asc"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.setResult("partitionName"); + if (max > 0) { + query.setRange(0, max); + } + Collection names = (Collection) query.execute(dbName, tableName); for (Iterator i = names.iterator(); i.hasNext();) { pns.add((String) i.next()); } + if (query != null) { + query.closeAll(); + } return pns; } @@ -1908,51 +1971,46 @@ public Partition getPartitionWithAuth(String dbName, String tblName, * has types of String, and if resultsCol is null, the types are MPartition. */ private Collection getPartitionPsQueryResults(String dbName, String tableName, - List part_vals, short max_parts, String resultsCol) + List part_vals, short max_parts, String resultsCol, QueryWrapper queryWrapper) throws MetaException, NoSuchObjectException { dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); Table table = getTable(dbName, tableName); - if (table == null) { throw new NoSuchObjectException(dbName + "." + tableName + " table not found"); } - List partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); if (part_vals.size() > numPartKeys) { throw new MetaException("Incorrect number of partition values"); } - partCols = partCols.subList(0, part_vals.size()); - //Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... // where partVal is either the escaped partition value given as input, // or a regex of the form ".*" - //This works because the "=" and "/" separating key names and partition key/values + // This works because the "=" and "/" separating key names and partition key/values // are not escaped. String partNameMatcher = Warehouse.makePartName(partCols, part_vals, ".*"); - //add ".*" to the regex to match anything else afterwards the partial spec. + // add ".*" to the regex to match anything else afterwards the partial spec. if (part_vals.size() < numPartKeys) { partNameMatcher += ".*"; } - - Query q = pm.newQuery(MPartition.class); + Query query = queryWrapper.query = pm.newQuery(MPartition.class); StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); queryFilter.append(" && table.tableName == tableName"); queryFilter.append(" && partitionName.matches(partialRegex)"); - q.setFilter(queryFilter.toString()); - q.declareParameters("java.lang.String dbName, " + - "java.lang.String tableName, java.lang.String partialRegex"); - - if( max_parts >= 0 ) { - //User specified a row limit, set it on the Query - q.setRange(0, max_parts); + query.setFilter(queryFilter.toString()); + query.declareParameters("java.lang.String dbName, " + + "java.lang.String tableName, java.lang.String partialRegex"); + if (max_parts >= 0) { + // User specified a row limit, set it on the Query + query.setRange(0, max_parts); } if (resultsCol != null && !resultsCol.isEmpty()) { - q.setResult(resultsCol); + query.setResult(resultsCol); } - return (Collection) q.execute(dbName, tableName, partNameMatcher); + return (Collection) query.execute(dbName, tableName, partNameMatcher); } @Override @@ -1961,11 +2019,13 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, throws MetaException, InvalidObjectException, NoSuchObjectException { List partitions = new ArrayList(); boolean success = false; + QueryWrapper queryWrapper = new QueryWrapper(); + try { openTransaction(); LOG.debug("executing listPartitionNamesPsWithAuth"); Collection parts = getPartitionPsQueryResults(db_name, tbl_name, - part_vals, max_parts, null); + part_vals, max_parts, null, queryWrapper); MTable mtbl = getMTable(db_name, tbl_name); for (Object o : parts) { Partition part = convertToPart((MPartition) o); @@ -1985,6 +2045,7 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, if (!success) { rollbackTransaction(); } + queryWrapper.close(); } return partitions; } @@ -1994,11 +2055,13 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, List part_vals, short max_parts) throws MetaException, NoSuchObjectException { List partitionNames = new ArrayList(); boolean success = false; + QueryWrapper queryWrapper = new QueryWrapper(); + try { openTransaction(); LOG.debug("Executing listPartitionNamesPs"); Collection names = getPartitionPsQueryResults(dbName, tableName, - part_vals, max_parts, "partitionName"); + part_vals, max_parts, "partitionName", queryWrapper); for (Object o : names) { partitionNames.add((String) o); } @@ -2007,14 +2070,13 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, if (!success) { rollbackTransaction(); } + queryWrapper.close(); } return partitionNames; } // TODO:pc implement max - private List listMPartitions(String dbName, String tableName, - int max) { - + private List listMPartitions(String dbName, String tableName, int max, QueryWrapper queryWrapper) { boolean success = false; List mparts = null; try { @@ -2022,11 +2084,10 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, LOG.debug("Executing listMPartitions"); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - Query query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2"); + Query query = queryWrapper.query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); query.setOrdering("partitionName ascending"); - if(max > 0) { + if (max > 0) { query.setRange(0, max); } mparts = (List) query.execute(tableName, dbName); @@ -2216,8 +2277,8 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, private List getPartitionsViaOrmFilter(Table table, ExpressionTree tree, short maxParts, boolean isValidatedFilter) throws MetaException { Map params = new HashMap(); - String jdoFilter = makeQueryFilterString( - table.getDbName(), table, tree, params, isValidatedFilter); + String jdoFilter = + makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter); if (jdoFilter == null) { assert !isValidatedFilter; return null; @@ -2227,14 +2288,11 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, // User specified a row limit, set it on the Query query.setRange(0, maxParts); } - String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); - @SuppressWarnings("unchecked") List mparts = (List) query.executeWithMap(params); - LOG.debug("Done executing query for getPartitionsViaOrmFilter"); pm.retrieveAll(mparts); // TODO: why is this inconsistent with what we get by names? LOG.debug("Done retrieving all objects for getPartitionsViaOrmFilter"); @@ -2243,10 +2301,6 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, return results; } - private static class Out { - public T val; - } - /** * Gets partition names from the table via ORM (JDOQL) name filter. * @param dbName Database name. @@ -2259,16 +2313,19 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, if (partNames.isEmpty()) { return new ArrayList(); } - Out query = new Out(); - List mparts = null; - try { - mparts = getMPartitionsViaOrmFilter(dbName, tblName, partNames, query); - return convertToParts(dbName, tblName, mparts); - } finally { - if (query.val != null) { - query.val.closeAll(); - } + ObjectPair> queryWithParams = + getPartQueryWithParams(dbName, tblName, partNames); + Query query = queryWithParams.getFirst(); + query.setResultClass(MPartition.class); + query.setClass(MPartition.class); + query.setOrdering("partitionName ascending"); + @SuppressWarnings("unchecked") + List mparts = (List)query.executeWithMap(queryWithParams.getSecond()); + List partitions = convertToParts(dbName, tblName, mparts); + if (query != null) { + query.closeAll(); } + return partitions; } private void dropPartitionsNoTxn(String dbName, String tblName, List partNames) { @@ -2304,27 +2361,15 @@ private void dropPartitionsNoTxn(String dbName, String tblName, List par sd.setCD(null); } } + if (query != null) { + query.closeAll(); + } return candidateCds; } - private List getMPartitionsViaOrmFilter(String dbName, - String tblName, List partNames, Out out) { - ObjectPair> queryWithParams = - getPartQueryWithParams(dbName, tblName, partNames); - Query query = out.val = queryWithParams.getFirst(); - query.setResultClass(MPartition.class); - query.setClass(MPartition.class); - query.setOrdering("partitionName ascending"); - - @SuppressWarnings("unchecked") - List result = (List)query.executeWithMap(queryWithParams.getSecond()); - return result; - } - - private ObjectPair> getPartQueryWithParams( - String dbName, String tblName, List partNames) { - StringBuilder sb = new StringBuilder( - "table.tableName == t1 && table.database.name == t2 && ("); + private ObjectPair> getPartQueryWithParams(String dbName, + String tblName, List partNames) { + StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 && ("); int n = 0; Map params = new HashMap(); for (Iterator itr = partNames.iterator(); itr.hasNext();) { @@ -2337,16 +2382,13 @@ private void dropPartitionsNoTxn(String dbName, String tblName, List par } sb.setLength(sb.length() - 4); // remove the last " || " sb.append(')'); - Query query = pm.newQuery(); query.setFilter(sb.toString()); - LOG.debug(" JDOQL filter is " + sb.toString()); params.put("t1", HiveStringUtils.normalizeIdentifier(tblName)); params.put("t2", HiveStringUtils.normalizeIdentifier(dbName)); - query.declareParameters(makeParameterDeclarationString(params)); - return new ObjectPair>(query, params); + return new ObjectPair>(query, params); } @Override @@ -2668,6 +2710,7 @@ private String makeParameterDeclarationStringObj(Map params) { public List listTableNamesByFilter(String dbName, String filter, short maxTables) throws MetaException { boolean success = false; + Query query = null; List tableNames = new ArrayList(); try { openTransaction(); @@ -2675,7 +2718,7 @@ private String makeParameterDeclarationStringObj(Map params) { dbName = HiveStringUtils.normalizeIdentifier(dbName); Map params = new HashMap(); String queryFilterString = makeQueryFilterString(dbName, null, filter, params); - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); query.declareImports("import java.lang.String"); query.setResult("tableName"); query.setResultClass(java.lang.String.class); @@ -2684,14 +2727,14 @@ private String makeParameterDeclarationStringObj(Map params) { } LOG.debug("filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString); for (Entry entry : params.entrySet()) { - LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() + - " class: " + entry.getValue().getClass().getName()); + LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() + " class: " + + entry.getValue().getClass().getName()); } String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setFilter(queryFilterString); - Collection names = (Collection) query.executeWithMap(params); - //have to emulate "distinct", otherwise tables with the same name may be returned + Collection names = (Collection)query.executeWithMap(params); + // have to emulate "distinct", otherwise tables with the same name may be returned Set tableNamesSet = new HashSet(); for (Iterator i = names.iterator(); i.hasNext();) { tableNamesSet.add((String) i.next()); @@ -2700,58 +2743,54 @@ private String makeParameterDeclarationStringObj(Map params) { LOG.debug("Done executing query for listTableNamesByFilter"); success = commitTransaction(); LOG.debug("Done retrieving all objects for listTableNamesByFilter"); - } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return tableNames; } @Override - public List listPartitionNamesByFilter(String dbName, String tableName, - String filter, short maxParts) throws MetaException { + public List listPartitionNamesByFilter(String dbName, String tableName, String filter, + short maxParts) throws MetaException { boolean success = false; + Query query = null; List partNames = new ArrayList(); try { openTransaction(); LOG.debug("Executing listMPartitionNamesByFilter"); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - MTable mtable = getMTable(dbName, tableName); - if( mtable == null ) { + if (mtable == null) { // To be consistent with the behavior of listPartitionNames, if the // table or db does not exist, we return an empty list return partNames; } Map params = new HashMap(); String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params); - Query query = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where " + queryFilterString); - - if( maxParts >= 0 ) { - //User specified a row limit, set it on the Query + query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where " + queryFilterString); + if (maxParts >= 0) { + // User specified a row limit, set it on the Query query.setRange(0, maxParts); } - - LOG.debug("Filter specified is " + filter + "," + - " JDOQL filter is " + queryFilterString); + LOG.debug("Filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString); LOG.debug("Parms is " + params); - String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); query.setResult("partitionName"); - Collection names = (Collection) query.executeWithMap(params); partNames = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { partNames.add((String) i.next()); } - LOG.debug("Done executing query for listMPartitionNamesByFilter"); success = commitTransaction(); LOG.debug("Done retrieving all objects for listMPartitionNamesByFilter"); @@ -2759,6 +2798,9 @@ private String makeParameterDeclarationStringObj(Map params) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return partNames; } @@ -2962,10 +3004,12 @@ private void removeUnusedColumnDescriptor(MColumnDescriptor oldCD) { } boolean success = false; + QueryWrapper queryWrapper = new QueryWrapper(); + try { openTransaction(); LOG.debug("execute removeUnusedColumnDescriptor"); - List referencedSDs = listStorageDescriptorsWithCD(oldCD, 1); + List referencedSDs = listStorageDescriptorsWithCD(oldCD, 1, queryWrapper); //if no other SD references this CD, we can throw it out. if (referencedSDs != null && referencedSDs.isEmpty()) { pm.retrieve(oldCD); @@ -2977,6 +3021,7 @@ private void removeUnusedColumnDescriptor(MColumnDescriptor oldCD) { if (!success) { rollbackTransaction(); } + queryWrapper.close(); } } @@ -3005,21 +3050,22 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { * @param maxSDs the maximum number of SDs to return * @return a list of storage descriptors */ - private List listStorageDescriptorsWithCD(MColumnDescriptor oldCD, - long maxSDs) { + private List listStorageDescriptorsWithCD( + MColumnDescriptor oldCD, + long maxSDs, + QueryWrapper queryWrapper) { boolean success = false; List sds = null; try { openTransaction(); LOG.debug("Executing listStorageDescriptorsWithCD"); - Query query = pm.newQuery(MStorageDescriptor.class, - "this.cd == inCD"); + Query query = queryWrapper.query = pm.newQuery(MStorageDescriptor.class, "this.cd == inCD"); query.declareParameters("MColumnDescriptor inCD"); - if(maxSDs >= 0) { - //User specified a row limit, set it on the Query + if (maxSDs >= 0) { + // User specified a row limit, set it on the Query query.setRange(0, maxSDs); } - sds = (List) query.execute(oldCD); + sds = (List)query.execute(oldCD); LOG.debug("Done executing query for listStorageDescriptorsWithCD"); pm.retrieveAll(sds); success = commitTransaction(); @@ -3096,9 +3142,11 @@ public boolean dropIndex(String dbName, String origTableName, String indexName) return success; } - private MIndex getMIndex(String dbName, String originalTblName, String indexName) throws MetaException { + private MIndex getMIndex(String dbName, String originalTblName, String indexName) + throws MetaException { MIndex midx = null; boolean commited = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -3108,19 +3156,23 @@ private MIndex getMIndex(String dbName, String originalTblName, String indexName commited = commitTransaction(); return null; } - - Query query = pm.newQuery(MIndex.class, - "origTable.tableName == t1 && origTable.database.name == t2 && indexName == t3"); + query = + pm.newQuery(MIndex.class, + "origTable.tableName == t1 && origTable.database.name == t2 && indexName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); - midx = (MIndex) query.execute(originalTblName, dbName, - HiveStringUtils.normalizeIdentifier(indexName)); + midx = + (MIndex) query.execute(originalTblName, dbName, + HiveStringUtils.normalizeIdentifier(indexName)); pm.retrieve(midx); commited = commitTransaction(); } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return midx; } @@ -3161,64 +3213,55 @@ private Index convertToIndex(MIndex mIndex) throws MetaException { public List getIndexes(String dbName, String origTableName, int max) throws MetaException { boolean success = false; + Query query = null; try { + LOG.debug("Executing getIndexes"); openTransaction(); - List mIndexList = listMIndexes(dbName, origTableName, max); - List indexes = new ArrayList(mIndexList.size()); - for (MIndex midx : mIndexList) { - indexes.add(this.convertToIndex(midx)); - } - success = commitTransaction(); - return indexes; - } finally { - if (!success) { - rollbackTransaction(); - } - } - } - private List listMIndexes(String dbName, String origTableName, - int max) { - boolean success = false; - List mindexes = null; - try { - openTransaction(); - LOG.debug("Executing listMIndexes"); dbName = HiveStringUtils.normalizeIdentifier(dbName); origTableName = HiveStringUtils.normalizeIdentifier(origTableName); - Query query = pm.newQuery(MIndex.class, - "origTable.tableName == t1 && origTable.database.name == t2"); + query = + pm.newQuery(MIndex.class, "origTable.tableName == t1 && origTable.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mindexes = (List) query.execute(origTableName, dbName); - LOG.debug("Done executing query for listMIndexes"); - pm.retrieveAll(mindexes); + List mIndexes = (List) query.execute(origTableName, dbName); + pm.retrieveAll(mIndexes); + + List indexes = new ArrayList(mIndexes.size()); + for (MIndex mIdx : mIndexes) { + indexes.add(this.convertToIndex(mIdx)); + } success = commitTransaction(); - LOG.debug("Done retrieving all objects for listMIndexes"); + LOG.debug("Done retrieving all objects for getIndexes"); + + return indexes; } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - return mindexes; } @Override - public List listIndexNames(String dbName, String origTableName, - short max) throws MetaException { + public List listIndexNames(String dbName, String origTableName, short max) + throws MetaException { List pns = new ArrayList(); boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listIndexNames"); dbName = HiveStringUtils.normalizeIdentifier(dbName); origTableName = HiveStringUtils.normalizeIdentifier(origTableName); - Query q = pm.newQuery( - "select indexName from org.apache.hadoop.hive.metastore.model.MIndex " - + "where origTable.database.name == t1 && origTable.tableName == t2 " - + "order by indexName asc"); - q.declareParameters("java.lang.String t1, java.lang.String t2"); - q.setResult("indexName"); - Collection names = (Collection) q.execute(dbName, origTableName); + query = + pm.newQuery("select indexName from org.apache.hadoop.hive.metastore.model.MIndex " + + "where origTable.database.name == t1 && origTable.tableName == t2 " + + "order by indexName asc"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.setResult("indexName"); + Collection names = (Collection) query.execute(dbName, origTableName); for (Iterator i = names.iterator(); i.hasNext();) { pns.add((String) i.next()); } @@ -3227,6 +3270,9 @@ private Index convertToIndex(MIndex mIndex) throws MetaException { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return pns; } @@ -3243,8 +3289,7 @@ public boolean addRole(String roleName, String ownerName) throw new InvalidObjectException("Role " + roleName + " already exists."); } int now = (int)(System.currentTimeMillis()/1000); - MRole mRole = new MRole(roleName, now, - ownerName); + MRole mRole = new MRole(roleName, now, ownerName); pm.makePersistent(mRole); commited = commitTransaction(); success = true; @@ -3334,13 +3379,16 @@ public boolean revokeRole(Role role, String userName, PrincipalType principalTyp return success; } - private MRoleMap getMSecurityUserRoleMap(String userName, - PrincipalType principalType, String roleName) { + private MRoleMap getMSecurityUserRoleMap(String userName, PrincipalType principalType, + String roleName) { MRoleMap mRoleMember = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2 && role.roleName == t3"); + query = + pm.newQuery(MRoleMap.class, + "principalName == t1 && principalType == t2 && role.roleName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); mRoleMember = (MRoleMap) query.executeWithArray(userName, principalType.toString(), roleName); @@ -3350,6 +3398,9 @@ private MRoleMap getMSecurityUserRoleMap(String userName, if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mRoleMember; } @@ -3358,6 +3409,7 @@ private MRoleMap getMSecurityUserRoleMap(String userName, public boolean removeRole(String roleName) throws MetaException, NoSuchObjectException { boolean success = false; + QueryWrapper queryWrapper = new QueryWrapper(); try { openTransaction(); MRole mRol = getMRole(roleName); @@ -3370,10 +3422,11 @@ public boolean removeRole(String roleName) throws MetaException, pm.deletePersistentAll(roleMap); } List roleMember = listMSecurityPrincipalMembershipRole(mRol - .getRoleName(), PrincipalType.ROLE); + .getRoleName(), PrincipalType.ROLE, queryWrapper); if (roleMember.size() > 0) { pm.deletePersistentAll(roleMember); } + queryWrapper.close(); // then remove all the grants List userGrants = listPrincipalGlobalGrants( mRol.getRoleName(), PrincipalType.ROLE); @@ -3381,30 +3434,36 @@ public boolean removeRole(String roleName) throws MetaException, pm.deletePersistentAll(userGrants); } List dbGrants = listPrincipalAllDBGrant(mRol - .getRoleName(), PrincipalType.ROLE); + .getRoleName(), PrincipalType.ROLE, queryWrapper); if (dbGrants.size() > 0) { pm.deletePersistentAll(dbGrants); } + queryWrapper.close(); List tabPartGrants = listPrincipalAllTableGrants( - mRol.getRoleName(), PrincipalType.ROLE); + mRol.getRoleName(), PrincipalType.ROLE, queryWrapper); if (tabPartGrants.size() > 0) { pm.deletePersistentAll(tabPartGrants); } + queryWrapper.close(); List partGrants = listPrincipalAllPartitionGrants( - mRol.getRoleName(), PrincipalType.ROLE); + mRol.getRoleName(), PrincipalType.ROLE, queryWrapper); if (partGrants.size() > 0) { pm.deletePersistentAll(partGrants); } + queryWrapper.close(); List tblColumnGrants = listPrincipalAllTableColumnGrants( - mRol.getRoleName(), PrincipalType.ROLE); + mRol.getRoleName(), PrincipalType.ROLE, queryWrapper); if (tblColumnGrants.size() > 0) { pm.deletePersistentAll(tblColumnGrants); } + queryWrapper.close(); List partColumnGrants = listPrincipalAllPartitionColumnGrants( - mRol.getRoleName(), PrincipalType.ROLE); + mRol.getRoleName(), PrincipalType.ROLE, queryWrapper); if (partColumnGrants.size() > 0) { pm.deletePersistentAll(partColumnGrants); } + queryWrapper.close(); + // finally remove the role pm.deletePersistent(mRol); } @@ -3413,6 +3472,8 @@ public boolean removeRole(String roleName) throws MetaException, if (!success) { rollbackTransaction(); } + + queryWrapper.close(); } return success; } @@ -3461,66 +3522,62 @@ private void getAllRoleAncestors(Set processedRoleNames, List @SuppressWarnings("unchecked") @Override - public List listRoles(String principalName, - PrincipalType principalType) { + public List listRoles(String principalName, PrincipalType principalType) { boolean success = false; - List mRoleMember = null; + Query query = null; + List mRoleMember = new ArrayList(); + try { - openTransaction(); LOG.debug("Executing listRoles"); - Query query = pm - .newQuery( - MRoleMap.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); + + openTransaction(); + query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); query.setUnique(false); - mRoleMember = (List) query.executeWithArray( - principalName, principalType.toString()); - LOG.debug("Done executing query for listMSecurityUserRoleMap"); - pm.retrieveAll(mRoleMember); + List mRoles = + (List) query.executeWithArray(principalName, principalType.toString()); + pm.retrieveAll(mRoles); success = commitTransaction(); - LOG.debug("Done retrieving all objects for listMSecurityUserRoleMap"); + + mRoleMember.addAll(mRoles); + + LOG.debug("Done retrieving all objects for listRoles"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } if (principalType == PrincipalType.USER) { // All users belong to public role implicitly, add that role - if (mRoleMember == null) { - mRoleMember = new ArrayList(); - } else { - mRoleMember = new ArrayList(mRoleMember); - } MRole publicRole = new MRole(HiveMetaStore.PUBLIC, 0, HiveMetaStore.PUBLIC); - mRoleMember.add(new MRoleMap(principalName, principalType.toString(), publicRole, 0, - null, null, false)); + mRoleMember.add(new MRoleMap(principalName, principalType.toString(), publicRole, 0, null, + null, false)); } - return mRoleMember; + return mRoleMember; } @SuppressWarnings("unchecked") private List listMSecurityPrincipalMembershipRole(final String roleName, - final PrincipalType principalType) { + final PrincipalType principalType, + QueryWrapper queryWrapper) { boolean success = false; List mRoleMemebership = null; try { - openTransaction(); LOG.debug("Executing listMSecurityPrincipalMembershipRole"); - Query query = pm.newQuery(MRoleMap.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); + + openTransaction(); + Query query = queryWrapper.query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); mRoleMemebership = (List) query.execute(roleName, principalType.toString()); - LOG - .debug("Done executing query for listMSecurityPrincipalMembershipRole"); pm.retrieveAll(mRoleMemebership); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listMSecurityPrincipalMembershipRole"); + + LOG.debug("Done retrieving all objects for listMSecurityPrincipalMembershipRole"); } finally { if (!success) { rollbackTransaction(); @@ -3543,9 +3600,10 @@ public Role getRole(String roleName) throws NoSuchObjectException { private MRole getMRole(String roleName) { MRole mrole = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MRole.class, "roleName == t1"); + query = pm.newQuery(MRole.class, "roleName == t1"); query.declareParameters("java.lang.String t1"); query.setUnique(true); mrole = (MRole) query.execute(roleName); @@ -3555,6 +3613,9 @@ private MRole getMRole(String roleName) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mrole; } @@ -3562,13 +3623,14 @@ private MRole getMRole(String roleName) { @Override public List listRoleNames() { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listAllRoleNames"); - Query query = pm.newQuery("select roleName from org.apache.hadoop.hive.metastore.model.MRole"); + query = pm.newQuery("select roleName from org.apache.hadoop.hive.metastore.model.MRole"); query.setResult("roleName"); Collection names = (Collection) query.execute(); - List roleNames = new ArrayList(); + List roleNames = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { roleNames.add((String) i.next()); } @@ -3578,6 +3640,9 @@ private MRole getMRole(String roleName) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -4386,49 +4451,61 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) @Override public List listRoleMembers(String roleName) { boolean success = false; - List mRoleMemeberList = null; + Query query = null; + List mRoleMemeberList = new ArrayList(); try { + LOG.debug("Executing listRoleMembers"); + openTransaction(); - LOG.debug("Executing listMSecurityUserRoleMember"); - Query query = pm.newQuery(MRoleMap.class, - "role.roleName == t1"); + query = pm.newQuery(MRoleMap.class, "role.roleName == t1"); query.declareParameters("java.lang.String t1"); query.setUnique(false); - mRoleMemeberList = (List) query.execute( - roleName); - LOG.debug("Done executing query for listMSecurityUserRoleMember"); - pm.retrieveAll(mRoleMemeberList); + List mRoles = (List) query.execute(roleName); + pm.retrieveAll(mRoles); success = commitTransaction(); - LOG.debug("Done retrieving all objects for listMSecurityUserRoleMember"); + + mRoleMemeberList.addAll(mRoles); + + LOG.debug("Done retrieving all objects for listRoleMembers"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mRoleMemeberList; } @SuppressWarnings("unchecked") @Override - public List listPrincipalGlobalGrants(String principalName, PrincipalType principalType) { + public List listPrincipalGlobalGrants(String principalName, + PrincipalType principalType) { boolean commited = false; - List userNameDbPriv = null; + Query query = null; + List userNameDbPriv = new ArrayList(); try { + List mPrivs = null; openTransaction(); if (principalName != null) { - Query query = pm.newQuery(MGlobalPrivilege.class, - "principalName == t1 && principalType == t2 "); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - userNameDbPriv = (List) query - .executeWithArray(principalName, principalType.toString()); - pm.retrieveAll(userNameDbPriv); + query = pm.newQuery(MGlobalPrivilege.class, "principalName == t1 && principalType == t2 "); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mPrivs = (List) query + .executeWithArray(principalName, principalType.toString()); + pm.retrieveAll(mPrivs); } commited = commitTransaction(); + if (mPrivs != null) { + userNameDbPriv.addAll(mPrivs); + } } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return userNameDbPriv; } @@ -4436,9 +4513,10 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) @Override public List listGlobalGrantsAll() { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MGlobalPrivilege.class); + query = pm.newQuery(MGlobalPrivilege.class); List userNameDbPriv = (List) query.execute(); pm.retrieveAll(userNameDbPriv); commited = commitTransaction(); @@ -4447,6 +4525,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -4470,25 +4551,32 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) public List listPrincipalDBGrants(String principalName, PrincipalType principalType, String dbName) { boolean success = false; - List mSecurityDBList = null; + Query query = null; + List mSecurityDBList = new ArrayList(); dbName = HiveStringUtils.normalizeIdentifier(dbName); - try { - openTransaction(); LOG.debug("Executing listPrincipalDBGrants"); - Query query = pm.newQuery(MDBPrivilege.class, - "principalName == t1 && principalType == t2 && database.name == t3"); - query - .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - mSecurityDBList = (List) query.executeWithArray(principalName, principalType.toString(), dbName); - LOG.debug("Done executing query for listPrincipalDBGrants"); - pm.retrieveAll(mSecurityDBList); + + openTransaction(); + query = + pm.newQuery(MDBPrivilege.class, + "principalName == t1 && principalType == t2 && database.name == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + List mPrivs = + (List) query.executeWithArray(principalName, principalType.toString(), + dbName); + pm.retrieveAll(mPrivs); success = commitTransaction(); + + mSecurityDBList.addAll(mPrivs); LOG.debug("Done retrieving all objects for listPrincipalDBGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityDBList; } @@ -4496,12 +4584,22 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) @Override public List listPrincipalDBGrantsAll( String principalName, PrincipalType principalType) { - return convertDB(listPrincipalAllDBGrant(principalName, principalType)); + QueryWrapper queryWrapper = new QueryWrapper(); + try { + return convertDB(listPrincipalAllDBGrant(principalName, principalType, queryWrapper)); + } finally { + queryWrapper.close(); + } } @Override public List listDBGrantsAll(String dbName) { - return convertDB(listDatabaseGrants(dbName)); + QueryWrapper queryWrapper = new QueryWrapper(); + try { + return convertDB(listDatabaseGrants(dbName, queryWrapper)); + } finally { + queryWrapper.close(); + } } private List convertDB(List privs) { @@ -4522,26 +4620,28 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) } @SuppressWarnings("unchecked") - private List listPrincipalAllDBGrant( - String principalName, PrincipalType principalType) { + private List listPrincipalAllDBGrant(String principalName, + PrincipalType principalType, + QueryWrapper queryWrapper) { boolean success = false; + Query query = null; List mSecurityDBList = null; try { - openTransaction(); LOG.debug("Executing listPrincipalAllDBGrant"); + + openTransaction(); if (principalName != null && principalType != null) { - Query query = pm.newQuery(MDBPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityDBList = (List) query.execute(principalName, principalType.toString()); + query = queryWrapper.query = pm.newQuery(MDBPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityDBList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MDBPrivilege.class); + query = queryWrapper.query = pm.newQuery(MDBPrivilege.class); mSecurityDBList = (List) query.execute(); } - LOG.debug("Done executing query for listPrincipalAllDBGrant"); pm.retrieveAll(mSecurityDBList); success = commitTransaction(); + LOG.debug("Done retrieving all objects for listPrincipalAllDBGrant"); } finally { if (!success) { @@ -4552,91 +4652,101 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) } @SuppressWarnings("unchecked") - public List listAllTableGrants(String dbName, - String tableName) { + public List listAllTableGrants(String dbName, String tableName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - List mSecurityTabList = null; + List mSecurityTabList = new ArrayList(); tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); try { - openTransaction(); LOG.debug("Executing listAllTableGrants"); + + openTransaction(); String queryStr = "table.tableName == t1 && table.database.name == t2"; - Query query = pm.newQuery( - MTablePrivilege.class, queryStr); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - mSecurityTabList = (List) query - .executeWithArray(tableName, dbName); + query = pm.newQuery(MTablePrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + List mPrivs = (List) query.executeWithArray(tableName, dbName); LOG.debug("Done executing query for listAllTableGrants"); - pm.retrieveAll(mSecurityTabList); + pm.retrieveAll(mPrivs); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listAllTableGrants"); + + mSecurityTabList.addAll(mPrivs); + + LOG.debug("Done retrieving all objects for listAllTableGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabList; } @SuppressWarnings("unchecked") - public List listTableAllPartitionGrants(String dbName, - String tableName) { + public List listTableAllPartitionGrants(String dbName, String tableName) { tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); boolean success = false; - List mSecurityTabPartList = null; + Query query = null; + List mSecurityTabPartList = new ArrayList(); try { - openTransaction(); LOG.debug("Executing listTableAllPartitionGrants"); + + openTransaction(); String queryStr = "partition.table.tableName == t1 && partition.table.database.name == t2"; - Query query = pm.newQuery( - MPartitionPrivilege.class, queryStr); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query - .executeWithArray(tableName, dbName); - LOG.debug("Done executing query for listTableAllPartitionGrants"); - pm.retrieveAll(mSecurityTabPartList); + query = pm.newQuery(MPartitionPrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + List mPrivs = (List) query.executeWithArray(tableName, dbName); + pm.retrieveAll(mPrivs); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listTableAllPartitionGrants"); + + mSecurityTabPartList.addAll(mPrivs); + + LOG.debug("Done retrieving all objects for listTableAllPartitionGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @SuppressWarnings("unchecked") - public List listTableAllColumnGrants(String dbName, - String tableName) { + public List listTableAllColumnGrants(String dbName, String tableName) { boolean success = false; - List mTblColPrivilegeList = null; + Query query = null; + List mTblColPrivilegeList = new ArrayList(); tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - try { - openTransaction(); LOG.debug("Executing listTableAllColumnGrants"); + + openTransaction(); String queryStr = "table.tableName == t1 && table.database.name == t2"; - Query query = pm.newQuery(MTableColumnPrivilege.class, queryStr); + query = pm.newQuery(MTableColumnPrivilege.class, queryStr); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mTblColPrivilegeList = (List) query - .executeWithArray(tableName, dbName); - LOG.debug("Done executing query for listTableAllColumnGrants"); - pm.retrieveAll(mTblColPrivilegeList); + List mPrivs = + (List) query.executeWithArray(tableName, dbName); + pm.retrieveAll(mPrivs); success = commitTransaction(); + + mTblColPrivilegeList.addAll(mPrivs); + LOG.debug("Done retrieving all objects for listTableAllColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mTblColPrivilegeList; } @@ -4645,26 +4755,32 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) public List listTableAllPartitionColumnGrants(String dbName, String tableName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - - List mSecurityColList = null; + List mSecurityColList = new ArrayList(); try { - openTransaction(); LOG.debug("Executing listTableAllPartitionColumnGrants"); + + openTransaction(); String queryStr = "partition.table.tableName == t1 && partition.table.database.name == t2"; - Query query = pm.newQuery(MPartitionColumnPrivilege.class, queryStr); + query = pm.newQuery(MPartitionColumnPrivilege.class, queryStr); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColList = (List) query - .executeWithArray(tableName, dbName); - LOG.debug("Done executing query for listTableAllPartitionColumnGrants"); - pm.retrieveAll(mSecurityColList); + List mPrivs = + (List) query.executeWithArray(tableName, dbName); + pm.retrieveAll(mPrivs); success = commitTransaction(); + + mSecurityColList.addAll(mPrivs); + LOG.debug("Done retrieving all objects for listTableAllPartitionColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @@ -4704,19 +4820,17 @@ public void dropPartitionAllColumnGrantsNoTxn( } @SuppressWarnings("unchecked") - private List listDatabaseGrants(String dbName) { + private List listDatabaseGrants(String dbName, QueryWrapper queryWrapper) { dbName = HiveStringUtils.normalizeIdentifier(dbName); - boolean success = false; + try { - openTransaction(); LOG.debug("Executing listDatabaseGrants"); - Query query = pm.newQuery(MDBPrivilege.class, - "database.name == t1"); + + openTransaction(); + Query query = queryWrapper.query = pm.newQuery(MDBPrivilege.class, "database.name == t1"); query.declareParameters("java.lang.String t1"); - List mSecurityDBList = (List) query - .executeWithArray(dbName); - LOG.debug("Done executing query for listDatabaseGrants"); + List mSecurityDBList = (List) query.executeWithArray(dbName); pm.retrieveAll(mSecurityDBList); success = commitTransaction(); LOG.debug("Done retrieving all objects for listDatabaseGrants"); @@ -4792,162 +4906,181 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listAllTableGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName) { + public List listAllTableGrants(String principalName, + PrincipalType principalType, String dbName, String tableName) { tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - boolean success = false; - List mSecurityTabPartList = null; + Query query = null; + List mSecurityTabPartList = new ArrayList(); try { openTransaction(); LOG.debug("Executing listAllTableGrants"); - Query query = pm.newQuery( - MTablePrivilege.class, + query = + pm.newQuery(MTablePrivilege.class, "principalName == t1 && principalType == t2 && table.tableName == t3 && table.database.name == t4"); - query.declareParameters( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); - mSecurityTabPartList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, dbName); - LOG.debug("Done executing query for listAllTableGrants"); - pm.retrieveAll(mSecurityTabPartList); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); + List mPrivs = + (List) query.executeWithArray(principalName, principalType.toString(), + tableName, dbName); + pm.retrieveAll(mPrivs); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listAllTableGrants"); + + mSecurityTabPartList.addAll(mPrivs); + + LOG.debug("Done retrieving all objects for listAllTableGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @SuppressWarnings("unchecked") @Override - public List listPrincipalPartitionGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String partName) { + public List listPrincipalPartitionGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - - List mSecurityTabPartList = null; + List mSecurityTabPartList = new ArrayList(); try { - openTransaction(); - LOG.debug("Executing listMSecurityPrincipalPartitionGrant"); - Query query = pm.newQuery( - MPartitionPrivilege.class, - "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + - "&& partition.table.database.name == t4 && partition.partitionName == t5"); - query.declareParameters( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4, " + - "java.lang.String t5"); - mSecurityTabPartList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, dbName, partName); - LOG.debug("Done executing query for listMSecurityPrincipalPartitionGrant"); + LOG.debug("Executing listPrincipalPartitionGrants"); - pm.retrieveAll(mSecurityTabPartList); + openTransaction(); + query = + pm.newQuery(MPartitionPrivilege.class, + "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + + "&& partition.table.database.name == t4 && partition.partitionName == t5"); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4, " + + "java.lang.String t5"); + List mPrivs = + (List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, partName); + pm.retrieveAll(mPrivs); success = commitTransaction(); - LOG.debug("Done retrieving all objects for listMSecurityPrincipalPartitionGrant"); + + mSecurityTabPartList.addAll(mPrivs); + + LOG.debug("Done retrieving all objects for listPrincipalPartitionGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @SuppressWarnings("unchecked") @Override - public List listPrincipalTableColumnGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String columnName) { + public List listPrincipalTableColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String columnName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); columnName = HiveStringUtils.normalizeIdentifier(columnName); - List mSecurityColList = null; + List mSecurityColList = new ArrayList(); try { - openTransaction(); LOG.debug("Executing listPrincipalTableColumnGrants"); - String queryStr = "principalName == t1 && principalType == t2 && " + - "table.tableName == t3 && table.database.name == t4 && columnName == t5 "; - Query query = pm.newQuery(MTableColumnPrivilege.class, queryStr); - query - .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + - "java.lang.String t4, java.lang.String t5"); - mSecurityColList = (List) query.executeWithArray( - principalName, principalType.toString(), tableName, dbName, columnName); - LOG.debug("Done executing query for listPrincipalTableColumnGrants"); - pm.retrieveAll(mSecurityColList); + + openTransaction(); + String queryStr = + "principalName == t1 && principalType == t2 && " + + "table.tableName == t3 && table.database.name == t4 && columnName == t5 "; + query = pm.newQuery(MTableColumnPrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + + "java.lang.String t4, java.lang.String t5"); + List mPrivs = + (List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, columnName); + pm.retrieveAll(mPrivs); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalTableColumnGrants"); + + mSecurityColList.addAll(mPrivs); + + LOG.debug("Done retrieving all objects for listPrincipalTableColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @Override @SuppressWarnings("unchecked") - public List listPrincipalPartitionColumnGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String partitionName, String columnName) { + public List listPrincipalPartitionColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partitionName, + String columnName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); columnName = HiveStringUtils.normalizeIdentifier(columnName); - - List mSecurityColList = null; + List mSecurityColList = new ArrayList(); try { - openTransaction(); LOG.debug("Executing listPrincipalPartitionColumnGrants"); - Query query = pm - .newQuery( + + openTransaction(); + query = pm.newQuery( MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + - "&& partition.table.database.name == t4 && partition.partitionName == t5 && columnName == t6"); - query - .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + - "java.lang.String t4, java.lang.String t5, java.lang.String t6"); + "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + + "&& partition.table.database.name == t4 && partition.partitionName == t5 && columnName == t6"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + + "java.lang.String t4, java.lang.String t5, java.lang.String t6"); + List mPrivs = + (List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, partitionName, columnName); + pm.retrieveAll(mPrivs); + success = commitTransaction(); - mSecurityColList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, - dbName, partitionName, columnName); - LOG.debug("Done executing query for listPrincipalPartitionColumnGrants"); - pm.retrieveAll(mSecurityColList); + mSecurityColList.addAll(mPrivs); - success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalPartitionColumnGrants"); + LOG.debug("Done retrieving all objects for listPrincipalPartitionColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @Override - public List listPrincipalPartitionColumnGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalPartitionColumnGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionColumnGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MPartitionColumnPrivilege.class, + "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.executeWithArray(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.executeWithArray(principalName, + principalType.toString()); } else { - Query query = pm.newQuery(MPartitionColumnPrivilege.class); + query = pm.newQuery(MPartitionColumnPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalPartitionColumnGrantsAll"); @@ -4960,23 +5093,29 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPartitionColumnGrantsAll( - String dbName, String tableName, String partitionName, String columnName) { + public List listPartitionColumnGrantsAll(String dbName, String tableName, + String partitionName, String columnName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPartitionColumnGrantsAll"); - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "partition.table.tableName == t3 && partition.table.database.name == t4 && " + - "partition.partitionName == t5 && columnName == t6"); - query.declareParameters( - "java.lang.String t3, java.lang.String t4, java.lang.String t5, java.lang.String t6"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, partitionName, columnName); + query = + pm.newQuery(MPartitionColumnPrivilege.class, + "partition.table.tableName == t3 && partition.table.database.name == t4 && " + + "partition.partitionName == t5 && columnName == t6"); + query + .declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5, java.lang.String t6"); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, + partitionName, columnName); LOG.debug("Done executing query for listPartitionColumnGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertPartCols(mSecurityTabPartList); @@ -4987,6 +5126,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllTableGrants( - String principalName, PrincipalType principalType) { + String principalName, PrincipalType principalType, QueryWrapper queryWrapper) { boolean success = false; List mSecurityTabPartList = null; try { - openTransaction(); LOG.debug("Executing listPrincipalAllTableGrants"); - Query query = pm.newQuery(MTablePrivilege.class, + + openTransaction(); + Query query = queryWrapper.query = pm.newQuery(MTablePrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); mSecurityTabPartList = (List) query.execute( principalName, principalType.toString()); - LOG - .debug("Done executing query for listPrincipalAllTableGrants"); pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalAllTableGrants"); + + LOG.debug("Done retrieving all objects for listPrincipalAllTableGrants"); } finally { if (!success) { rollbackTransaction(); @@ -5038,21 +5179,21 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalTableGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalTableGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllTableGrants"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MTablePrivilege.class, - "principalName == t1 && principalType == t2"); + query = pm.newQuery(MTablePrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query.execute( - principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MTablePrivilege.class); + query = pm.newQuery(MTablePrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalAllTableGrants"); @@ -5065,20 +5206,24 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listTableGrantsAll(String dbName, String tableName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listTableGrantsAll"); - Query query = pm.newQuery(MTablePrivilege.class, - "table.tableName == t1 && table.database.name == t2"); + query = + pm.newQuery(MTablePrivilege.class, "table.tableName == t1 && table.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName); LOG.debug("Done executing query for listTableGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertTable(mSecurityTabPartList); @@ -5089,6 +5234,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllPartitionGrants( - String principalName, PrincipalType principalType) { + private List listPrincipalAllPartitionGrants(String principalName, + PrincipalType principalType, QueryWrapper queryWrapper) { boolean success = false; List mSecurityTabPartList = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllPartitionGrants"); - Query query = pm.newQuery(MPartitionPrivilege.class, - "principalName == t1 && principalType == t2"); + Query query = queryWrapper.query = pm.newQuery(MPartitionPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query.execute( - principalName, principalType.toString()); - LOG - .debug("Done executing query for listPrincipalAllPartitionGrants"); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalAllPartitionGrants"); + LOG.debug("Done retrieving all objects for listPrincipalAllPartitionGrants"); } finally { if (!success) { rollbackTransaction(); @@ -5139,21 +5283,22 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalPartitionGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalPartitionGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MPartitionPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MPartitionPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.execute(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MPartitionPrivilege.class); + query = pm.newQuery(MPartitionPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalPartitionGrantsAll"); @@ -5166,22 +5311,27 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPartitionGrantsAll( - String dbName, String tableName, String partitionName) { + public List listPartitionGrantsAll(String dbName, String tableName, + String partitionName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionGrantsAll"); - Query query = pm.newQuery(MPartitionPrivilege.class, - "partition.table.tableName == t3 && partition.table.database.name == t4 && " + - "partition.partitionName == t5"); + query = + pm.newQuery(MPartitionPrivilege.class, + "partition.table.tableName == t3 && partition.table.database.name == t4 && " + + "partition.partitionName == t5"); query.declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, partitionName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, partitionName); LOG.debug("Done executing query for listPrincipalPartitionGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertPartition(mSecurityTabPartList); @@ -5192,6 +5342,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllTableColumnGrants( - String principalName, PrincipalType principalType) { + private List listPrincipalAllTableColumnGrants(String principalName, + PrincipalType principalType, QueryWrapper queryWrapper) { boolean success = false; + List mSecurityColumnList = null; try { - openTransaction(); LOG.debug("Executing listPrincipalAllTableColumnGrants"); - Query query = pm.newQuery(MTableColumnPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColumnList = (List) query.execute( - principalName, principalType.toString()); - LOG.debug("Done executing query for listPrincipalAllTableColumnGrants"); + + openTransaction(); + Query query = queryWrapper.query = + pm.newQuery(MTableColumnPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityColumnList = + (List) query.execute(principalName, principalType.toString()); pm.retrieveAll(mSecurityColumnList); success = commitTransaction(); + LOG.debug("Done retrieving all objects for listPrincipalAllTableColumnGrants"); } finally { if (!success) { @@ -5242,22 +5396,23 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalTableColumnGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalTableColumnGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalTableColumnGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MTableColumnPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MTableColumnPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.execute(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MTableColumnPrivilege.class); + query = pm.newQuery(MTableColumnPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalTableColumnGrantsAll"); @@ -5270,21 +5425,26 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listTableColumnGrantsAll( - String dbName, String tableName, String columnName) { + public List listTableColumnGrantsAll(String dbName, String tableName, + String columnName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalTableColumnGrantsAll"); - Query query = pm.newQuery(MTableColumnPrivilege.class, - "table.tableName == t3 && table.database.name == t4 && columnName == t5"); + query = + pm.newQuery(MTableColumnPrivilege.class, + "table.tableName == t3 && table.database.name == t4 && columnName == t5"); query.declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, columnName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, columnName); LOG.debug("Done executing query for listPrincipalTableColumnGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertTableCols(mSecurityTabPartList); @@ -5295,6 +5455,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllPartitionColumnGrants( - String principalName, PrincipalType principalType) { + String principalName, PrincipalType principalType, QueryWrapper queryWrapper) { boolean success = false; List mSecurityColumnList = null; try { - openTransaction(); LOG.debug("Executing listPrincipalAllTableColumnGrants"); - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColumnList = (List) query.execute( - principalName, principalType.toString()); - LOG.debug("Done executing query for listPrincipalAllTableColumnGrants"); + + openTransaction(); + Query query = queryWrapper.query = + pm.newQuery(MPartitionColumnPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityColumnList = + (List) query.execute(principalName, principalType.toString()); pm.retrieveAll(mSecurityColumnList); success = commitTransaction(); + LOG.debug("Done retrieving all objects for listPrincipalAllTableColumnGrants"); } finally { if (!success) { @@ -5347,29 +5510,37 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List partName, PartitionEventType evtType) throws UnknownTableException, MetaException, InvalidPartitionException, UnknownPartitionException { - - Collection partEvents; boolean success = false; - LOG.debug("Begin Executing isPartitionMarkedForEvent"); - try{ - openTransaction(); - Query query = pm.newQuery(MPartitionEvent.class, "dbName == t1 && tblName == t2 && partName == t3 && eventType == t4"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4"); - Table tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. - if(null == tbl) { - throw new UnknownTableException("Table: "+ tblName + " is not found."); - } - partEvents = (Collection) query.executeWithArray(dbName, tblName, getPartitionStr(tbl, partName), evtType.getValue()); - pm.retrieveAll(partEvents); - success = commitTransaction(); - LOG.debug("Done executing isPartitionMarkedForEvent"); - } finally{ + Query query = null; + + try { + LOG.debug("Begin Executing isPartitionMarkedForEvent"); + + openTransaction(); + query = pm.newQuery(MPartitionEvent.class, + "dbName == t1 && tblName == t2 && partName == t3 && eventType == t4"); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4"); + Table tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. + if (null == tbl) { + throw new UnknownTableException("Table: " + tblName + " is not found."); + } + Collection partEvents = + (Collection) query.executeWithArray(dbName, tblName, + getPartitionStr(tbl, partName), evtType.getValue()); + pm.retrieveAll(partEvents); + success = commitTransaction(); + + LOG.debug("Done executing isPartitionMarkedForEvent"); + return (partEvents != null && !partEvents.isEmpty()) ? true : false; + } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - return (partEvents != null && !partEvents.isEmpty()) ? true : false; - } @Override @@ -5419,15 +5590,16 @@ private String getPartitionStr(Table tbl, Map partName) throws In * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public Collection executeJDOQLSelect(String query) { + public Collection executeJDOQLSelect(String queryStr, QueryWrapper queryWrapper) { boolean committed = false; Collection result = null; try { openTransaction(); - Query q = pm.newQuery(query); - result = (Collection) q.execute(); + Query query = queryWrapper.query = pm.newQuery(queryStr); + result = ((Collection) query.execute()); committed = commitTransaction(); + if (committed) { return result; } else { @@ -5447,14 +5619,14 @@ private String getPartitionStr(Table tbl, Map partName) throws In * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public long executeJDOQLUpdate(String query) { + public long executeJDOQLUpdate(String queryStr) { boolean committed = false; long numUpdated = 0; - + Query query = null; try { openTransaction(); - Query q = pm.newQuery(query); - numUpdated = (Long) q.execute(); + query = pm.newQuery(queryStr); + numUpdated = (Long) query.execute(); committed = commitTransaction(); if (committed) { return numUpdated; @@ -5465,6 +5637,9 @@ public long executeJDOQLUpdate(String query) { if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5477,15 +5652,14 @@ public long executeJDOQLUpdate(String query) { */ public Set listFSRoots() { boolean committed = false; + Query query = null; Set fsRoots = new HashSet(); - try { openTransaction(); - Query query = pm.newQuery(MDatabase.class); + query = pm.newQuery(MDatabase.class); List mDBs = (List) query.execute(); pm.retrieveAll(mDBs); - - for (MDatabase mDB:mDBs) { + for (MDatabase mDB : mDBs) { fsRoots.add(mDB.getLocationUri()); } committed = commitTransaction(); @@ -5498,6 +5672,9 @@ public long executeJDOQLUpdate(String query) { if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5572,17 +5749,17 @@ public void setUpdateLocations(Map updateLocations) { */ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boolean dryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateMDatabaseURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MDatabase.class); + query = pm.newQuery(MDatabase.class); List mDBs = (List) query.execute(); pm.retrieveAll(mDBs); - for(MDatabase mDB:mDBs) { + for (MDatabase mDB : mDBs) { URI locationURI = null; String location = mDB.getLocationUri(); try { @@ -5611,6 +5788,9 @@ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boole if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5675,16 +5855,16 @@ private void updatePropURIHelper(URI oldLoc, URI newLoc, String tblPropKey, bool * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, boolean - isDryRun) { + public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, + boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap<>(); List badRecords = new ArrayList<>(); UpdatePropURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); List mTbls = (List) query.execute(); pm.retrieveAll(mTbls); @@ -5701,6 +5881,9 @@ public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPr if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5712,20 +5895,19 @@ public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPr * */ @Deprecated - public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, - URI newLoc, String tblPropKey, boolean isDryRun) { + public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, URI newLoc, + String tblPropKey, boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdatePropURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MStorageDescriptor.class); + query = pm.newQuery(MStorageDescriptor.class); List mSDSs = (List) query.execute(); pm.retrieveAll(mSDSs); - - for(MStorageDescriptor mSDS:mSDSs) { + for (MStorageDescriptor mSDS : mSDSs) { updatePropURIHelper(oldLoc, newLoc, tblPropKey, isDryRun, badRecords, updateLocations, mSDS.getParameters()); } @@ -5734,11 +5916,14 @@ public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, retVal = new UpdatePropURIRetVal(badRecords, updateLocations); } return retVal; - } finally { - if (!committed) { - rollbackTransaction(); - } - } + } finally { + if (!committed) { + rollbackTransaction(); + } + if (query != null) { + query.closeAll(); + } + } } public class UpdateMStorageDescriptorTblURIRetVal { @@ -5775,20 +5960,19 @@ public void setUpdateLocations(Map updateLocations) { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, URI newLoc, - boolean isDryRun) { + public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, + URI newLoc, boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateMStorageDescriptorTblURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MStorageDescriptor.class); + query = pm.newQuery(MStorageDescriptor.class); List mSDSs = (List) query.execute(); pm.retrieveAll(mSDSs); - - for(MStorageDescriptor mSDS:mSDSs) { + for (MStorageDescriptor mSDS : mSDSs) { URI locationURI = null; String location = mSDS.getLocation(); try { @@ -5814,10 +5998,13 @@ public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI o } return retVal; } finally { - if (!committed) { - rollbackTransaction(); - } - } + if (!committed) { + rollbackTransaction(); + } + if (query != null) { + query.closeAll(); + } + } } public class UpdateSerdeURIRetVal { @@ -5854,19 +6041,18 @@ public void setUpdateLocations(Map updateLocations) { * */ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeProp, - boolean isDryRun) { + boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateSerdeURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MSerDeInfo.class); + query = pm.newQuery(MSerDeInfo.class); List mSerdes = (List) query.execute(); pm.retrieveAll(mSerdes); - - for(MSerDeInfo mSerde:mSerdes) { + for (MSerDeInfo mSerde : mSerdes) { if (mSerde.getParameters().containsKey(serdeProp)) { String schemaLoc = mSerde.getParameters().get(serdeProp); URI schemaLocURI = null; @@ -5897,6 +6083,9 @@ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeP if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5905,19 +6094,24 @@ private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mSt String dbName = mStatsObj.getDbName(); String tableName = mStatsObj.getTableName(); String colName = mStatsObj.getColName(); + QueryWrapper queryWrapper = new QueryWrapper(); - LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName - + " colName=" + colName); - validateTableCols(table, Lists.newArrayList(colName)); + try { + LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName + + " colName=" + colName); + validateTableCols(table, Lists.newArrayList(colName)); - List oldStats = - getMTableColumnStatistics(table, Lists.newArrayList(colName)); + List oldStats = + getMTableColumnStatistics(table, Lists.newArrayList(colName), queryWrapper); - if (!oldStats.isEmpty()) { - assert oldStats.size() == 1; - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); - } else { - pm.makePersistent(mStatsObj); + if (!oldStats.isEmpty()) { + assert oldStats.size() == 1; + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); + } else { + pm.makePersistent(mStatsObj); + } + } finally { + queryWrapper.close(); } } @@ -5947,16 +6141,21 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, " for which stats gathering is requested doesn't exist."); } - List oldStats = getMPartitionColumnStatistics( - table, Lists.newArrayList(partName), Lists.newArrayList(colName)); - if (!oldStats.isEmpty()) { - assert oldStats.size() == 1; - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); + QueryWrapper queryWrapper = new QueryWrapper(); + try { + List oldStats = getMPartitionColumnStatistics( + table, Lists.newArrayList(partName), Lists.newArrayList(colName), queryWrapper); + if (!oldStats.isEmpty()) { + assert oldStats.size() == 1; + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0)); } else { pm.makePersistent(mStatsObj); } + } finally { + queryWrapper.close(); + } } - + @Override public boolean updateTableColumnStatistics(ColumnStatistics colStats) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { @@ -6017,15 +6216,16 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List getMTableColumnStatistics( - Table table, List colNames) throws MetaException { + private List getMTableColumnStatistics(Table table, List colNames, QueryWrapper queryWrapper) + throws MetaException { boolean committed = false; - openTransaction(); + try { + openTransaction(); + List result = null; validateTableCols(table, colNames); - - Query query = pm.newQuery(MTableColumnStatistics.class); + Query query = queryWrapper.query = pm.newQuery(MTableColumnStatistics.class); String filter = "tableName == t1 && dbName == t2 && ("; String paramStr = "java.lang.String t1, java.lang.String t2"; Object[] params = new Object[colNames.size() + 2]; @@ -6042,15 +6242,15 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List) query.executeWithArray(params); pm.retrieveAll(result); if (result.size() > colNames.size()) { - throw new MetaException( - "Unexpected " + result.size() + " statistics for " + colNames.size() + " columns"); + throw new MetaException("Unexpected " + result.size() + " statistics for " + + colNames.size() + " columns"); } committed = commitTransaction(); return result; } catch (Exception ex) { LOG.error("Error retrieving statistics via jdo", ex); if (ex instanceof MetaException) { - throw (MetaException)ex; + throw (MetaException) ex; } throw new MetaException(ex.getMessage()); } finally { @@ -6094,8 +6294,11 @@ protected ColumnStatistics getSqlResult(GetHelper ctx) throws } @Override protected ColumnStatistics getJdoResult( - GetHelper ctx) throws MetaException, NoSuchObjectException { - List mStats = getMTableColumnStatistics(getTable(), colNames); + GetHelper ctx) throws MetaException { + QueryWrapper queryWrapper = new QueryWrapper(); + + try { + List mStats = getMTableColumnStatistics(getTable(), colNames, queryWrapper); if (mStats.isEmpty()) return null; // LastAnalyzed is stored per column, but thrift object has it per multiple columns. // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now. @@ -6109,6 +6312,9 @@ protected ColumnStatistics getJdoResult( Deadline.checkTimeout(); } return new ColumnStatistics(desc, statObjs); + } finally { + queryWrapper.close(); + } } }.run(true); } @@ -6132,32 +6338,37 @@ protected ColumnStatistics getJdoResult( @Override protected List getJdoResult( GetHelper> ctx) throws MetaException, NoSuchObjectException { - List mStats = - getMPartitionColumnStatistics(getTable(), partNames, colNames); - List result = new ArrayList( - Math.min(mStats.size(), partNames.size())); - String lastPartName = null; - List curList = null; - ColumnStatisticsDesc csd = null; - for (int i = 0; i <= mStats.size(); ++i) { - boolean isLast = i == mStats.size(); - MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i); - String partName = isLast ? null : (String)mStatsObj.getPartitionName(); - if (isLast || !partName.equals(lastPartName)) { - if (i != 0) { - result.add(new ColumnStatistics(csd, curList)); - } - if (isLast) { - continue; + QueryWrapper queryWrapper = new QueryWrapper(); + try { + List mStats = + getMPartitionColumnStatistics(getTable(), partNames, colNames, queryWrapper); + List result = new ArrayList( + Math.min(mStats.size(), partNames.size())); + String lastPartName = null; + List curList = null; + ColumnStatisticsDesc csd = null; + for (int i = 0; i <= mStats.size(); ++i) { + boolean isLast = i == mStats.size(); + MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i); + String partName = isLast ? null : (String)mStatsObj.getPartitionName(); + if (isLast || !partName.equals(lastPartName)) { + if (i != 0) { + result.add(new ColumnStatistics(csd, curList)); + } + if (isLast) { + continue; + } + csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj); + curList = new ArrayList(colNames.size()); } - csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj); - curList = new ArrayList(colNames.size()); + curList.add(StatObjectConverter.getPartitionColumnStatisticsObj(mStatsObj)); + lastPartName = partName; + Deadline.checkTimeout(); } - curList.add(StatObjectConverter.getPartitionColumnStatisticsObj(mStatsObj)); - lastPartName = partName; - Deadline.checkTimeout(); + return result; + } finally { + queryWrapper.close(); } - return result; } }.run(true); } @@ -6189,17 +6400,16 @@ protected String describeResult() { }.run(true); } - private List getMPartitionColumnStatistics( - Table table, List partNames, List colNames) - throws NoSuchObjectException, MetaException { + private List getMPartitionColumnStatistics(Table table, + List partNames, List colNames, + QueryWrapper queryWrapper) throws NoSuchObjectException, MetaException { boolean committed = false; - MPartitionColumnStatistics mStatsObj = null; + try { openTransaction(); // We are not going to verify SD for each partition. Just verify for the table. validateTableCols(table, colNames); - boolean foundCol = false; - Query query = pm.newQuery(MPartitionColumnStatistics.class); + Query query = queryWrapper.query = pm.newQuery(MPartitionColumnStatistics.class); String paramStr = "java.lang.String t1, java.lang.String t2"; String filter = "tableName == t1 && dbName == t2 && ("; Object[] params = new Object[colNames.size() + partNames.size() + 2]; @@ -6232,7 +6442,7 @@ protected String describeResult() { } catch (Exception ex) { LOG.error("Error retrieving statistics via jdo", ex); if (ex instanceof MetaException) { - throw (MetaException)ex; + throw (MetaException) ex; } throw new MetaException(ex.getMessage()); } finally { @@ -6252,127 +6462,112 @@ private void dropPartitionColumnStatisticsNoTxn( } @Override - public boolean deletePartitionColumnStatistics(String dbName, String tableName, - String partName, List partVals, String colName) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName, + List partVals, String colName) throws NoSuchObjectException, MetaException, + InvalidObjectException, InvalidInputException { boolean ret = false; - + Query query = null; if (dbName == null) { dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; } - if (tableName == null) { throw new InvalidInputException("Table name is null."); } - try { openTransaction(); MTable mTable = getMTable(dbName, tableName); MPartitionColumnStatistics mStatsObj; List mStatsObjColl; - if (mTable == null) { - throw new - NoSuchObjectException("Table " + tableName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Table " + tableName + + " for which stats deletion is requested doesn't exist"); } - - MPartition mPartition = - getMPartition(dbName, tableName, partVals); - + MPartition mPartition = getMPartition(dbName, tableName, partVals); if (mPartition == null) { - throw new - NoSuchObjectException("Partition " + partName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Partition " + partName + + " for which stats deletion is requested doesn't exist"); } - - Query query = pm.newQuery(MPartitionColumnStatistics.class); + query = pm.newQuery(MPartitionColumnStatistics.class); String filter; String parameters; - if (colName != null) { - filter = "partition.partitionName == t1 && dbName == t2 && tableName == t3 && " + - "colName == t4"; - parameters = "java.lang.String t1, java.lang.String t2, " + - "java.lang.String t3, java.lang.String t4"; + filter = + "partition.partitionName == t1 && dbName == t2 && tableName == t3 && " + + "colName == t4"; + parameters = + "java.lang.String t1, java.lang.String t2, " + + "java.lang.String t3, java.lang.String t4"; } else { filter = "partition.partitionName == t1 && dbName == t2 && tableName == t3"; parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; } - query.setFilter(filter); - query - .declareParameters(parameters); - + query.declareParameters(parameters); if (colName != null) { query.setUnique(true); - mStatsObj = (MPartitionColumnStatistics)query.executeWithArray(partName.trim(), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(colName)); + mStatsObj = + (MPartitionColumnStatistics) query.executeWithArray(partName.trim(), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(colName)); pm.retrieve(mStatsObj); - if (mStatsObj != null) { pm.deletePersistent(mStatsObj); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" +dbName + " table=" + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName + " partition=" + partName + " col=" + colName); } } else { - mStatsObjColl= (List)query.execute(partName.trim(), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(tableName)); + mStatsObjColl = + (List) query.execute(partName.trim(), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(tableName)); pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { pm.deletePersistentAll(mStatsObjColl); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + - " table=" + tableName + " partition" + partName); + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + + tableName + " partition" + partName); } } ret = commitTransaction(); - } catch(NoSuchObjectException e) { - rollbackTransaction(); - throw e; + } catch (NoSuchObjectException e) { + rollbackTransaction(); + throw e; } finally { if (!ret) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return ret; } @Override public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException - { + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { boolean ret = false; - + Query query = null; if (dbName == null) { dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; } - if (tableName == null) { throw new InvalidInputException("Table name is null."); } - try { openTransaction(); MTable mTable = getMTable(dbName, tableName); MTableColumnStatistics mStatsObj; - List mStatsObjColl; - + List mStatsObjColl; if (mTable == null) { - throw new - NoSuchObjectException("Table " + tableName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Table " + tableName + + " for which stats deletion is requested doesn't exist"); } - - Query query = pm.newQuery(MTableColumnStatistics.class); + query = pm.newQuery(MTableColumnStatistics.class); String filter; String parameters; - if (colName != null) { filter = "table.tableName == t1 && dbName == t2 && colName == t3"; parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; @@ -6382,44 +6577,45 @@ public boolean deleteTableColumnStatistics(String dbName, String tableName, Stri } query.setFilter(filter); - query - .declareParameters(parameters); - + query.declareParameters(parameters); if (colName != null) { query.setUnique(true); - mStatsObj = (MTableColumnStatistics)query.execute( - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(colName)); + mStatsObj = + (MTableColumnStatistics) query.execute(HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(colName)); pm.retrieve(mStatsObj); if (mStatsObj != null) { pm.deletePersistent(mStatsObj); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" +dbName + " table=" + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName + " col=" + colName); } } else { - mStatsObjColl= (List)query.execute( - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(dbName)); + mStatsObjColl = + (List) query.execute( + HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(dbName)); pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { pm.deletePersistentAll(mStatsObjColl); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + - " table=" + tableName); + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + + tableName); } } ret = commitTransaction(); - } catch(NoSuchObjectException e) { - rollbackTransaction(); - throw e; + } catch (NoSuchObjectException e) { + rollbackTransaction(); + throw e; } finally { if (!ret) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return ret; } @@ -6427,21 +6623,26 @@ public boolean deleteTableColumnStatistics(String dbName, String tableName, Stri @Override public long cleanupEvents() { boolean commited = false; + Query query = null; long delCnt; LOG.debug("Begin executing cleanupEvents"); - Long expiryTime = HiveConf.getTimeVar(getConf(), ConfVars.METASTORE_EVENT_EXPIRY_DURATION, TimeUnit.MILLISECONDS); + Long expiryTime = + HiveConf.getTimeVar(getConf(), ConfVars.METASTORE_EVENT_EXPIRY_DURATION, + TimeUnit.MILLISECONDS); Long curTime = System.currentTimeMillis(); try { openTransaction(); - Query query = pm.newQuery(MPartitionEvent.class,"curTime - eventTime > expiryTime"); + query = pm.newQuery(MPartitionEvent.class, "curTime - eventTime > expiryTime"); query.declareParameters("java.lang.Long curTime, java.lang.Long expiryTime"); delCnt = query.deletePersistentAll(curTime, expiryTime); commited = commitTransaction(); - } - finally { + } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } LOG.debug("Done executing cleanupEvents"); } return delCnt; @@ -6451,7 +6652,11 @@ private MDelegationToken getTokenFrom(String tokenId) { Query query = pm.newQuery(MDelegationToken.class, "tokenIdentifier == tokenId"); query.declareParameters("java.lang.String tokenId"); query.setUnique(true); - return (MDelegationToken)query.execute(tokenId); + MDelegationToken delegationToken = (MDelegationToken) query.execute(tokenId); + if (query != null) { + query.closeAll(); + } + return delegationToken; } @Override @@ -6523,28 +6728,31 @@ public String getToken(String tokenId) { @Override public List getAllTokenIdentifiers() { - LOG.debug("Begin executing getAllTokenIdentifiers"); boolean committed = false; - List tokens; - try{ + Query query = null; + List tokenIdents = new ArrayList(); + + try { openTransaction(); - Query query = pm.newQuery(MDelegationToken.class); - tokens = (List) query.execute(); + query = pm.newQuery(MDelegationToken.class); + List tokens = (List) query.execute(); pm.retrieveAll(tokens); committed = commitTransaction(); + + for (MDelegationToken token : tokens) { + tokenIdents.add(token.getTokenIdentifier()); + } + return tokenIdents; } finally { - if(!committed) { + LOG.debug("Done executing getAllTokenIdentifers with status : " + committed); + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - LOG.debug("Done executing getAllTokenIdentifers with status : " + committed); - List tokenIdents = new ArrayList(tokens.size()); - - for (MDelegationToken token : tokens) { - tokenIdents.add(token.getTokenIdentifier()); - } - return tokenIdents; } @Override @@ -6573,21 +6781,25 @@ public int addMasterKey(String key) throws MetaException{ public void updateMasterKey(Integer id, String key) throws NoSuchObjectException, MetaException { LOG.debug("Begin executing updateMasterKey"); boolean committed = false; + Query query = null; MMasterKey masterKey; - try{ - openTransaction(); - Query query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey)query.execute(id); - if (null != masterKey) { - masterKey.setMasterKey(key); - } - committed = commitTransaction(); + try { + openTransaction(); + query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + masterKey.setMasterKey(key); + } + committed = commitTransaction(); } finally { - if(!committed) { + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } LOG.debug("Done executing updateMasterKey with status : " + committed); if (null == masterKey) { @@ -6602,21 +6814,25 @@ public void updateMasterKey(Integer id, String key) throws NoSuchObjectException public boolean removeMasterKey(Integer id) { LOG.debug("Begin executing removeMasterKey"); boolean success = false; + Query query = null; MMasterKey masterKey; - try{ - openTransaction(); - Query query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey)query.execute(id); - if (null != masterKey) { - pm.deletePersistent(masterKey); - } - success = commitTransaction(); + try { + openTransaction(); + query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + pm.deletePersistent(masterKey); + } + success = commitTransaction(); } finally { - if(!success) { + if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } LOG.debug("Done executing removeMasterKey with status : " + success); return (null != masterKey) && success; @@ -6626,25 +6842,29 @@ public boolean removeMasterKey(Integer id) { public String[] getMasterKeys() { LOG.debug("Begin executing getMasterKeys"); boolean committed = false; + Query query = null; List keys; - try{ + try { openTransaction(); - Query query = pm.newQuery(MMasterKey.class); + query = pm.newQuery(MMasterKey.class); keys = (List) query.execute(); pm.retrieveAll(keys); committed = commitTransaction(); + + String[] masterKeys = new String[keys.size()]; + for (int i = 0; i < keys.size(); i++) { + masterKeys[i] = keys.get(i).getMasterKey(); + } + return masterKeys; } finally { - if(!committed) { + LOG.debug("Done executing getMasterKeys with status : " + committed); + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - LOG.debug("Done executing getMasterKeys with status : " + committed); - String[] masterKeys = new String[keys.size()]; - - for (int i = 0; i < keys.size(); i++) { - masterKeys[i] = keys.get(i).getMasterKey(); - } - return masterKeys; } // compare hive version and metastore version @@ -6720,43 +6940,46 @@ public String getMetaStoreSchemaVersion() throws MetaException { } @SuppressWarnings("unchecked") - private MVersionTable getMSchemaVersion() - throws NoSuchObjectException, MetaException { + private MVersionTable getMSchemaVersion() throws NoSuchObjectException, MetaException { boolean committed = false; + Query query = null; List mVerTables = new ArrayList(); - try { openTransaction(); - Query query = pm.newQuery(MVersionTable.class); - + query = pm.newQuery(MVersionTable.class); try { - mVerTables = (List)query.execute(); + mVerTables = (List) query.execute(); pm.retrieveAll(mVerTables); } catch (JDODataStoreException e) { if (e.getCause() instanceof MissingTableException) { - throw new MetaException("Version table not found. " + - "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion()); + throw new MetaException("Version table not found. " + "The metastore is not upgraded to " + + MetaStoreSchemaInfo.getHiveSchemaVersion()); } else { throw e; } } committed = commitTransaction(); + if (mVerTables.isEmpty()) { + throw new NoSuchObjectException("No matching version found"); + } + if (mVerTables.size() > 1) { + String msg = "Metastore contains multiple versions (" + mVerTables.size() + ") "; + for (MVersionTable version : mVerTables) { + msg += + "[ version = " + version.getSchemaVersion() + ", comment = " + + version.getVersionComment() + " ] "; + } + throw new MetaException(msg.trim()); + } + return mVerTables.get(0); } finally { if (!committed) { rollbackTransaction(); } - } - if (mVerTables.isEmpty()) { - throw new NoSuchObjectException("No matching version found"); - } - if (mVerTables.size() > 1) { - String msg = "Metastore contains multiple versions (" + mVerTables.size() + ") "; - for (MVersionTable version : mVerTables) { - msg += "[ version = " + version.getSchemaVersion() + ", comment = " + version.getVersionComment() + " ] "; + if (query != null) { + query.closeAll(); } - throw new MetaException(msg.trim()); } - return mVerTables.get(0); } @Override @@ -6795,6 +7018,7 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro public boolean doesPartitionExist(String dbName, String tableName, List partVals) throws MetaException { boolean success = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -6806,22 +7030,24 @@ public boolean doesPartitionExist(String dbName, String tableName, List success = commitTransaction(); return false; } - - Query query = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.tableName == t1 && table.database.name == t2 && partitionName == t3"); + query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where table.tableName == t1 && table.database.name == t2 && partitionName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); query.setResult("partitionName"); - String name = Warehouse.makePartName( - convertToFieldSchemas(mtbl.getPartitionKeys()), partVals); - String result = (String)query.execute(tableName, dbName, name); + String name = + Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), partVals); + String result = (String) query.execute(tableName, dbName, name); success = commitTransaction(); return result != null; } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -6992,11 +7218,12 @@ public void dropFunction(String dbName, String funcName) throws MetaException, private MFunction getMFunction(String db, String function) { MFunction mfunc = null; boolean commited = false; + Query query = null; try { openTransaction(); db = HiveStringUtils.normalizeIdentifier(db); function = HiveStringUtils.normalizeIdentifier(function); - Query query = pm.newQuery(MFunction.class, "functionName == function && database.name == db"); + query = pm.newQuery(MFunction.class, "functionName == function && database.name == db"); query.declareParameters("java.lang.String function, java.lang.String db"); query.setUnique(true); mfunc = (MFunction) query.execute(function, db); @@ -7006,6 +7233,9 @@ private MFunction getMFunction(String db, String function) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mfunc; } @@ -7027,9 +7257,9 @@ public Function getFunction(String dbName, String funcName) throws MetaException } @Override - public List getFunctions(String dbName, String pattern) - throws MetaException { + public List getFunctions(String dbName, String pattern) throws MetaException { boolean commited = false; + Query query = null; List funcs = null; try { openTransaction(); @@ -7037,25 +7267,24 @@ public Function getFunction(String dbName, String funcName) throws MetaException // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = - "select functionName from org.apache.hadoop.hive.metastore.model.MFunction " - + "where database.name == dbName && ("; + String queryStr = + "select functionName from org.apache.hadoop.hive.metastore.model.MFunction " + + "where database.name == dbName && ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " functionName.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " functionName.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.declareParameters("java.lang.String dbName"); - q.setResult("functionName"); - q.setOrdering("functionName ascending"); - Collection names = (Collection) q.execute(dbName); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.declareParameters("java.lang.String dbName"); + query.setResult("functionName"); + query.setOrdering("functionName ascending"); + Collection names = (Collection) query.execute(dbName); funcs = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { funcs.add((String) i.next()); @@ -7065,6 +7294,9 @@ public Function getFunction(String dbName, String funcName) throws MetaException if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return funcs; } @@ -7072,13 +7304,14 @@ public Function getFunction(String dbName, String funcName) throws MetaException @Override public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) { boolean commited = false; + Query query = null; try { openTransaction(); long lastEvent = rqst.getLastEvent(); - Query query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); + query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); query.declareParameters("java.lang.Long lastEvent"); query.setOrdering("eventId ascending"); - Collection events = (Collection)query.execute(lastEvent); + Collection events = (Collection) query.execute(lastEvent); commited = commitTransaction(); if (events == null) { return null; @@ -7093,6 +7326,9 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq } return result; } finally { + if (query != null) { + query.closeAll(); + } if (!commited) { rollbackTransaction(); return null; @@ -7103,9 +7339,10 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq @Override public void addNotificationEvent(NotificationEvent entry) { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MNotificationNextId.class); + query = pm.newQuery(MNotificationNextId.class); Collection ids = (Collection) query.execute(); MNotificationNextId id = null; boolean needToPersistId; @@ -7118,26 +7355,31 @@ public void addNotificationEvent(NotificationEvent entry) { } entry.setEventId(id.getNextEventId()); id.incrementEventId(); - if (needToPersistId) pm.makePersistent(id); + if (needToPersistId) + pm.makePersistent(id); pm.makePersistent(translateThriftToDb(entry)); commited = commitTransaction(); } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @Override public void cleanNotificationEvents(int olderThan) { boolean commited = false; + Query query = null; try { openTransaction(); long tmp = System.currentTimeMillis() / 1000 - olderThan; - int tooOld = (tmp > Integer.MAX_VALUE) ? 0 : (int)tmp; - Query query = pm.newQuery(MNotificationLog.class, "eventTime < tooOld"); + int tooOld = (tmp > Integer.MAX_VALUE) ? 0 : (int) tmp; + query = pm.newQuery(MNotificationLog.class, "eventTime < tooOld"); query.declareParameters("java.lang.Integer tooOld"); - Collection toBeRemoved = (Collection)query.execute(tooOld); + Collection toBeRemoved = (Collection) query.execute(tooOld); if (toBeRemoved != null && toBeRemoved.size() > 0) { pm.deletePersistent(toBeRemoved); } @@ -7146,16 +7388,20 @@ public void cleanNotificationEvents(int olderThan) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @Override public CurrentNotificationEventId getCurrentNotificationEventId() { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MNotificationNextId.class); - Collection ids = (Collection)query.execute(); + query = pm.newQuery(MNotificationNextId.class); + Collection ids = (Collection) query.execute(); long id = 0; if (ids != null && ids.size() > 0) { id = ids.iterator().next().getNextEventId() - 1; @@ -7166,6 +7412,9 @@ public CurrentNotificationEventId getCurrentNotificationEventId() { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -7190,7 +7439,4 @@ private NotificationEvent translateDbToThrift(MNotificationLog dbEvent) { event.setMessage((dbEvent.getMessage())); return event; } - - - } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java index d0ff329..411ac21 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java @@ -149,16 +149,21 @@ private void executeJDOQLSelect(String query) { initObjectStore(hiveConf); System.out.println("Executing query: " + query); - Collection result = objStore.executeJDOQLSelect(query); - if (result != null) { - Iterator iter = result.iterator(); - while (iter.hasNext()) { - Object o = iter.next(); - System.out.println(o.toString()); + ObjectStore.QueryWrapper queryWrapper = new ObjectStore.QueryWrapper(); + try { + Collection result = objStore.executeJDOQLSelect(query, queryWrapper); + if (result != null) { + Iterator iter = result.iterator(); + while (iter.hasNext()) { + Object o = iter.next(); + System.out.println(o.toString()); + } + } else { + System.err.println("Encountered error during executeJDOQLSelect -" + + "commit of JDO transaction failed."); } - } else { - System.err.println("Encountered error during executeJDOQLSelect -" + - "commit of JDO transaction failed."); + } finally { + queryWrapper.close(); } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java new file mode 100644 index 0000000..a4f9f6c --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestObjectStore { + private ObjectStore objectStore = null; + + private static final String DB1 = "testobjectstoredb1"; + private static final String DB2 = "testobjectstoredb2"; + private static final String TABLE1 = "testobjectstoretable1"; + private static final String KEY1 = "testobjectstorekey1"; + private static final String KEY2 = "testobjectstorekey2"; + private static final String OWNER = "testobjectstoreowner"; + private static final String USER1 = "testobjectstoreuser1"; + private static final String ROLE1 = "testobjectstorerole1"; + private static final String ROLE2 = "testobjectstorerole2"; + + public static class MockPartitionExpressionProxy implements PartitionExpressionProxy { + @Override + public String convertExprToFilter(byte[] expr) throws MetaException { + return null; + } + + @Override + public boolean filterPartitionsByExpr(List partColumnNames, + List partColumnTypeInfos, byte[] expr, + String defaultPartitionName, List partitionNames) + throws MetaException { + return false; + } + } + + @Before + public void setUp() { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName()); + + objectStore = new ObjectStore(); + objectStore.setConf(conf); + + Deadline.registerIfNot(100000); + try { + objectStore.dropDatabase(DB1); + } catch (Exception e) { + } + try { + objectStore.dropDatabase(DB2); + } catch (Exception e) { + } + } + + @After + public void tearDown() { + } + + /** + * Test database operations + */ + @Test + public void testDatabaseOps() throws MetaException, InvalidObjectException, NoSuchObjectException { + Database db1 = new Database(DB1, "description", "locationurl", null); + Database db2 = new Database(DB2, "description", "locationurl", null); + objectStore.createDatabase(db1); + objectStore.createDatabase(db2); + + List databases = objectStore.getAllDatabases(); + Assert.assertEquals(2, databases.size()); + Assert.assertEquals(DB1, databases.get(0)); + Assert.assertEquals(DB2, databases.get(1)); + + objectStore.dropDatabase(DB1); + databases = objectStore.getAllDatabases(); + Assert.assertEquals(1, databases.size()); + Assert.assertEquals(DB2, databases.get(0)); + + objectStore.dropDatabase(DB2); + } + + /** + * Test table operations + */ + @Test + public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { + Database db1 = new Database(DB1, "description", "locationurl", null); + objectStore.createDatabase(db1); + StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); + HashMap params = new HashMap(); + params.put("EXTERNAL", "false"); + Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.createTable(tbl1); + + List tables = objectStore.getAllTables(DB1); + Assert.assertEquals(1, tables.size()); + Assert.assertEquals(TABLE1, tables.get(0)); + + Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.alterTable(DB1, TABLE1, newTbl1); + tables = objectStore.getTables(DB1, "new*"); + Assert.assertEquals(1, tables.size()); + Assert.assertEquals("new" + TABLE1, tables.get(0)); + + objectStore.dropTable(DB1, "new" + TABLE1); + tables = objectStore.getAllTables(DB1); + Assert.assertEquals(0, tables.size()); + + objectStore.dropDatabase(DB1); + } + + /** + * Tests partition operations + */ + @Test + public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { + Database db1 = new Database(DB1, "description", "locationurl", null); + objectStore.createDatabase(db1); + StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); + HashMap tableParams = new HashMap(); + tableParams.put("EXTERNAL", "false"); + FieldSchema partitionKey1 = new FieldSchema("Country", "String", ""); + FieldSchema partitionKey2 = new FieldSchema("State", "String", ""); + Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.createTable(tbl1); + HashMap partitionParams = new HashMap(); + partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true"); + List value1 = Arrays.asList("US", "CA"); + Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams); + objectStore.addPartition(part1); + List value2 = Arrays.asList("US", "MA"); + Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd, partitionParams); + objectStore.addPartition(part2); + + Deadline.startTimer("getPartition"); + List partitions = objectStore.getPartitions(DB1, TABLE1, 10); + Assert.assertEquals(2, partitions.size()); + Assert.assertEquals(111, partitions.get(0).getCreateTime()); + Assert.assertEquals(222, partitions.get(1).getCreateTime()); + + objectStore.dropPartition(DB1, TABLE1, value1); + partitions = objectStore.getPartitions(DB1, TABLE1, 10); + Assert.assertEquals(1, partitions.size()); + Assert.assertEquals(222, partitions.get(0).getCreateTime()); + + objectStore.dropPartition(DB1, TABLE1, value2); + objectStore.dropTable(DB1, TABLE1); + objectStore.dropDatabase(DB1); + } + + /** + * Test master keys operation + */ + @Test + public void testMasterKeyOps() throws MetaException, NoSuchObjectException { + int id1 = objectStore.addMasterKey(KEY1); + int id2 = objectStore.addMasterKey(KEY2); + + String[] keys = objectStore.getMasterKeys(); + Assert.assertEquals(2, keys.length); + Assert.assertEquals(KEY1, keys[0]); + Assert.assertEquals(KEY2, keys[1]); + + objectStore.updateMasterKey(id1, "new" + KEY1); + objectStore.updateMasterKey(id2, "new" + KEY2); + keys = objectStore.getMasterKeys(); + Assert.assertEquals(2, keys.length); + Assert.assertEquals("new" + KEY1, keys[0]); + Assert.assertEquals("new" + KEY2, keys[1]); + + objectStore.removeMasterKey(id1); + keys = objectStore.getMasterKeys(); + Assert.assertEquals(1, keys.length); + Assert.assertEquals("new" + KEY2, keys[0]); + + objectStore.removeMasterKey(id2); + } + + /** + * Test role operation + */ + @Test + public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchObjectException { + objectStore.addRole(ROLE1, OWNER); + objectStore.addRole(ROLE2, OWNER); + List roles = objectStore.listRoleNames(); + Assert.assertEquals(2, roles.size()); + Assert.assertEquals(ROLE2, roles.get(1)); + Role role1 = objectStore.getRole(ROLE1); + Assert.assertEquals(OWNER, role1.getOwnerName()); + objectStore.grantRole(role1, USER1, PrincipalType.USER, OWNER, PrincipalType.ROLE, true); + objectStore.revokeRole(role1, USER1, PrincipalType.USER, false); + objectStore.removeRole(ROLE1); + } +}