diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 417ecc8..111c36d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -533,10 +533,11 @@ public void createDatabase(Database db) throws InvalidObjectException, MetaExcep private MDatabase getMDatabase(String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; + Query query = null; try { openTransaction(); name = HiveStringUtils.normalizeIdentifier(name); - Query query = pm.newQuery(MDatabase.class, "name == dbname"); + query = pm.newQuery(MDatabase.class, "name == dbname"); query.declareParameters("java.lang.String dbname"); query.setUnique(true); mdb = (MDatabase) query.execute(name); @@ -546,6 +547,9 @@ private MDatabase getMDatabase(String name) throws NoSuchObjectException { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } if (mdb == null) { throw new NoSuchObjectException("There is no database named " + name); @@ -670,32 +674,31 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc return success; } - @Override public List getDatabases(String pattern) throws MetaException { boolean commited = false; List databases = null; + Query query = null; try { openTransaction(); // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = "select name from org.apache.hadoop.hive.metastore.model.MDatabase where ("; + String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MDatabase where ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " name.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " name.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.setResult("name"); - q.setOrdering("name ascending"); - Collection names = (Collection) q.execute(); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.setResult("name"); + query.setOrdering("name ascending"); + Collection names = (Collection) query.execute(); databases = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { databases.add((String) i.next()); @@ -705,6 +708,9 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return databases; } @@ -763,9 +769,10 @@ public boolean createType(Type type) { public Type getType(String typeName) { Type type = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MType.class, "name == typeName"); + query = pm.newQuery(MType.class, "name == typeName"); query.declareParameters("java.lang.String typeName"); query.setUnique(true); MType mtype = (MType) query.execute(typeName.trim()); @@ -778,6 +785,9 @@ public Type getType(String typeName) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return type; } @@ -785,9 +795,10 @@ public Type getType(String typeName) { @Override public boolean dropType(String typeName) { boolean success = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MType.class, "name == typeName"); + query = pm.newQuery(MType.class, "name == typeName"); query.declareParameters("java.lang.String typeName"); query.setUnique(true); MType type = (MType) query.execute(typeName.trim()); @@ -803,6 +814,9 @@ public boolean dropType(String typeName) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return success; } @@ -938,9 +952,9 @@ public Table getTable(String dbName, String tableName) throws MetaException { } @Override - public List getTables(String dbName, String pattern) - throws MetaException { + public List getTables(String dbName, String pattern) throws MetaException { boolean commited = false; + Query query = null; List tbls = null; try { openTransaction(); @@ -948,25 +962,24 @@ public Table getTable(String dbName, String tableName) throws MetaException { // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = - "select tableName from org.apache.hadoop.hive.metastore.model.MTable " - + "where database.name == dbName && ("; + String queryStr = + "select tableName from org.apache.hadoop.hive.metastore.model.MTable " + + "where database.name == dbName && ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " tableName.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " tableName.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.declareParameters("java.lang.String dbName"); - q.setResult("tableName"); - q.setOrdering("tableName ascending"); - Collection names = (Collection) q.execute(dbName); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.declareParameters("java.lang.String dbName"); + query.setResult("tableName"); + query.setOrdering("tableName ascending"); + Collection names = (Collection) query.execute(dbName); tbls = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { tbls.add((String) i.next()); @@ -976,6 +989,9 @@ public Table getTable(String dbName, String tableName) throws MetaException { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return tbls; } @@ -988,11 +1004,12 @@ public Table getTable(String dbName, String tableName) throws MetaException { private MTable getMTable(String db, String table) { MTable mtbl = null; boolean commited = false; + Query query = null; try { openTransaction(); db = HiveStringUtils.normalizeIdentifier(db); table = HiveStringUtils.normalizeIdentifier(table); - Query query = pm.newQuery(MTable.class, "tableName == table && database.name == db"); + query = pm.newQuery(MTable.class, "tableName == table && database.name == db"); query.declareParameters("java.lang.String table, java.lang.String db"); query.setUnique(true); mtbl = (MTable) query.execute(table, db); @@ -1002,20 +1019,24 @@ private MTable getMTable(String db, String table) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mtbl; } @Override - public List getTableObjectsByName(String db, List tbl_names) - throws MetaException, UnknownDBException { + public List
getTableObjectsByName(String db, List tbl_names) throws MetaException, + UnknownDBException { List
tables = new ArrayList
(); boolean committed = false; + Query dbExistsQuery = null; + Query query = null; try { openTransaction(); - db = HiveStringUtils.normalizeIdentifier(db); - Query dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); + dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); dbExistsQuery.declareParameters("java.lang.String db"); dbExistsQuery.setUnique(true); dbExistsQuery.setResult("name"); @@ -1028,7 +1049,7 @@ private MTable getMTable(String db, String table) { for (String t : tbl_names) { lowered_tbl_names.add(HiveStringUtils.normalizeIdentifier(t)); } - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); query.setFilter("database.name == db && tbl_names.contains(tableName)"); query.declareParameters("java.lang.String db, java.util.Collection tbl_names"); Collection mtables = (Collection) query.execute(db, lowered_tbl_names); @@ -1040,6 +1061,12 @@ private MTable getMTable(String db, String table) { if (!committed) { rollbackTransaction(); } + if (dbExistsQuery != null) { + dbExistsQuery.closeAll(); + } + if (query != null) { + query.closeAll(); + } } return tables; } @@ -1513,10 +1540,11 @@ public Partition getPartition(String dbName, String tableName, return part; } - private MPartition getMPartition(String dbName, String tableName, - List part_vals) throws MetaException { + private MPartition getMPartition(String dbName, String tableName, List part_vals) + throws MetaException { MPartition mpart = null; boolean commited = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -1528,10 +1556,11 @@ private MPartition getMPartition(String dbName, String tableName, } // Change the query to use part_vals instead of the name which is // redundant TODO: callers of this often get part_vals out of name for no reason... - String name = Warehouse.makePartName(convertToFieldSchemas(mtbl - .getPartitionKeys()), part_vals); - Query query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2 && partitionName == t3"); + String name = + Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals); + query = + pm.newQuery(MPartition.class, + "table.tableName == t1 && table.database.name == t2 && partitionName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); mpart = (MPartition) query.execute(tableName, dbName, name); @@ -1541,6 +1570,9 @@ private MPartition getMPartition(String dbName, String tableName, if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mpart; } @@ -1857,20 +1889,22 @@ public Partition getPartitionWithAuth(String dbName, String tblName, List pns = new ArrayList(); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - Query q = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.database.name == t1 && table.tableName == t2 " - + "order by partitionName asc"); - q.declareParameters("java.lang.String t1, java.lang.String t2"); - q.setResult("partitionName"); - - if(max > 0) { - q.setRange(0, max); - } - Collection names = (Collection) q.execute(dbName, tableName); + Query query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where table.database.name == t1 && table.tableName == t2 " + + "order by partitionName asc"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.setResult("partitionName"); + if (max > 0) { + query.setRange(0, max); + } + Collection names = (Collection) query.execute(dbName, tableName); for (Iterator i = names.iterator(); i.hasNext();) { pns.add((String) i.next()); } + if (query != null) { + query.closeAll(); + } return pns; } @@ -1895,46 +1929,44 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); Table table = getTable(dbName, tableName); - if (table == null) { throw new NoSuchObjectException(dbName + "." + tableName + " table not found"); } - List partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); if (part_vals.size() > numPartKeys) { throw new MetaException("Incorrect number of partition values"); } - partCols = partCols.subList(0, part_vals.size()); - //Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... // where partVal is either the escaped partition value given as input, // or a regex of the form ".*" - //This works because the "=" and "/" separating key names and partition key/values + // This works because the "=" and "/" separating key names and partition key/values // are not escaped. String partNameMatcher = Warehouse.makePartName(partCols, part_vals, ".*"); - //add ".*" to the regex to match anything else afterwards the partial spec. + // add ".*" to the regex to match anything else afterwards the partial spec. if (part_vals.size() < numPartKeys) { partNameMatcher += ".*"; } - - Query q = pm.newQuery(MPartition.class); + Query query = pm.newQuery(MPartition.class); StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); queryFilter.append(" && table.tableName == tableName"); queryFilter.append(" && partitionName.matches(partialRegex)"); - q.setFilter(queryFilter.toString()); - q.declareParameters("java.lang.String dbName, " + - "java.lang.String tableName, java.lang.String partialRegex"); - - if( max_parts >= 0 ) { - //User specified a row limit, set it on the Query - q.setRange(0, max_parts); + query.setFilter(queryFilter.toString()); + query.declareParameters("java.lang.String dbName, " + + "java.lang.String tableName, java.lang.String partialRegex"); + if (max_parts >= 0) { + // User specified a row limit, set it on the Query + query.setRange(0, max_parts); } if (resultsCol != null && !resultsCol.isEmpty()) { - q.setResult(resultsCol); + query.setResult(resultsCol); } - - return (Collection) q.execute(dbName, tableName, partNameMatcher); + Collection partItems = cloneQueryResult((Collection) query.execute(dbName, tableName, partNameMatcher)); + if (query != null) { + query.closeAll(); + } + return partItems; } @Override @@ -1994,24 +2026,22 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, } // TODO:pc implement max - private List listMPartitions(String dbName, String tableName, - int max) { - + private List listMPartitions(String dbName, String tableName, int max) { boolean success = false; + Query query = null; List mparts = null; try { openTransaction(); LOG.debug("Executing listMPartitions"); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - Query query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2"); + query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); query.setOrdering("partitionName ascending"); - if(max > 0) { + if (max > 0) { query.setRange(0, max); } - mparts = (List) query.execute(tableName, dbName); + mparts = cloneQueryResult((List) query.execute(tableName, dbName)); LOG.debug("Done executing query for listMPartitions"); pm.retrieveAll(mparts); success = commitTransaction(); @@ -2020,6 +2050,9 @@ private Collection getPartitionPsQueryResults(String dbName, String tableName, if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mparts; } @@ -2198,8 +2231,8 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, private List getPartitionsViaOrmFilter(Table table, ExpressionTree tree, short maxParts, boolean isValidatedFilter) throws MetaException { Map params = new HashMap(); - String jdoFilter = makeQueryFilterString( - table.getDbName(), table, tree, params, isValidatedFilter); + String jdoFilter = + makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter); if (jdoFilter == null) { assert !isValidatedFilter; return null; @@ -2209,14 +2242,11 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, // User specified a row limit, set it on the Query query.setRange(0, maxParts); } - String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); - @SuppressWarnings("unchecked") List mparts = (List) query.executeWithMap(params); - LOG.debug("Done executing query for getPartitionsViaOrmFilter"); pm.retrieveAll(mparts); // TODO: why is this inconsistent with what we get by names? LOG.debug("Done retrieving all objects for getPartitionsViaOrmFilter"); @@ -2241,16 +2271,8 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, if (partNames.isEmpty()) { return new ArrayList(); } - Out query = new Out(); - List mparts = null; - try { - mparts = getMPartitionsViaOrmFilter(dbName, tblName, partNames, query); - return convertToParts(dbName, tblName, mparts); - } finally { - if (query.val != null) { - query.val.closeAll(); - } - } + List mparts = getMPartitionsViaOrmFilter(dbName, tblName, partNames); + return convertToParts(dbName, tblName, mparts); } private void dropPartitionsNoTxn(String dbName, String tblName, List partNames) { @@ -2286,27 +2308,31 @@ private void dropPartitionsNoTxn(String dbName, String tblName, List par sd.setCD(null); } } + if (query != null) { + query.closeAll(); + } return candidateCds; } private List getMPartitionsViaOrmFilter(String dbName, - String tblName, List partNames, Out out) { + String tblName, List partNames) { ObjectPair> queryWithParams = getPartQueryWithParams(dbName, tblName, partNames); - Query query = out.val = queryWithParams.getFirst(); + Query query = queryWithParams.getFirst(); query.setResultClass(MPartition.class); query.setClass(MPartition.class); query.setOrdering("partitionName ascending"); - @SuppressWarnings("unchecked") - List result = (List)query.executeWithMap(queryWithParams.getSecond()); + List result = cloneQueryResult((List)query.executeWithMap(queryWithParams.getSecond())); + if (query != null) { + query.closeAll(); + } return result; } - private ObjectPair> getPartQueryWithParams( - String dbName, String tblName, List partNames) { - StringBuilder sb = new StringBuilder( - "table.tableName == t1 && table.database.name == t2 && ("); + private ObjectPair> getPartQueryWithParams(String dbName, + String tblName, List partNames) { + StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 && ("); int n = 0; Map params = new HashMap(); for (Iterator itr = partNames.iterator(); itr.hasNext();) { @@ -2319,16 +2345,13 @@ private void dropPartitionsNoTxn(String dbName, String tblName, List par } sb.setLength(sb.length() - 4); // remove the last " || " sb.append(')'); - Query query = pm.newQuery(); query.setFilter(sb.toString()); - LOG.debug(" JDOQL filter is " + sb.toString()); params.put("t1", HiveStringUtils.normalizeIdentifier(tblName)); params.put("t2", HiveStringUtils.normalizeIdentifier(dbName)); - query.declareParameters(makeParameterDeclarationString(params)); - return new ObjectPair>(query, params); + return new ObjectPair>(query, params); } @Override @@ -2650,6 +2673,7 @@ private String makeParameterDeclarationStringObj(Map params) { public List listTableNamesByFilter(String dbName, String filter, short maxTables) throws MetaException { boolean success = false; + Query query = null; List tableNames = new ArrayList(); try { openTransaction(); @@ -2657,7 +2681,7 @@ private String makeParameterDeclarationStringObj(Map params) { dbName = HiveStringUtils.normalizeIdentifier(dbName); Map params = new HashMap(); String queryFilterString = makeQueryFilterString(dbName, null, filter, params); - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); query.declareImports("import java.lang.String"); query.setResult("tableName"); query.setResultClass(java.lang.String.class); @@ -2666,14 +2690,14 @@ private String makeParameterDeclarationStringObj(Map params) { } LOG.debug("filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString); for (Entry entry : params.entrySet()) { - LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() + - " class: " + entry.getValue().getClass().getName()); + LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() + " class: " + + entry.getValue().getClass().getName()); } String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setFilter(queryFilterString); - Collection names = (Collection) query.executeWithMap(params); - //have to emulate "distinct", otherwise tables with the same name may be returned + Collection names = (Collection)query.executeWithMap(params); + // have to emulate "distinct", otherwise tables with the same name may be returned Set tableNamesSet = new HashSet(); for (Iterator i = names.iterator(); i.hasNext();) { tableNamesSet.add((String) i.next()); @@ -2682,58 +2706,54 @@ private String makeParameterDeclarationStringObj(Map params) { LOG.debug("Done executing query for listTableNamesByFilter"); success = commitTransaction(); LOG.debug("Done retrieving all objects for listTableNamesByFilter"); - } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return tableNames; } @Override - public List listPartitionNamesByFilter(String dbName, String tableName, - String filter, short maxParts) throws MetaException { + public List listPartitionNamesByFilter(String dbName, String tableName, String filter, + short maxParts) throws MetaException { boolean success = false; + Query query = null; List partNames = new ArrayList(); try { openTransaction(); LOG.debug("Executing listMPartitionNamesByFilter"); dbName = HiveStringUtils.normalizeIdentifier(dbName); tableName = HiveStringUtils.normalizeIdentifier(tableName); - MTable mtable = getMTable(dbName, tableName); - if( mtable == null ) { + if (mtable == null) { // To be consistent with the behavior of listPartitionNames, if the // table or db does not exist, we return an empty list return partNames; } Map params = new HashMap(); String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params); - Query query = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where " + queryFilterString); - - if( maxParts >= 0 ) { - //User specified a row limit, set it on the Query + query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where " + queryFilterString); + if (maxParts >= 0) { + // User specified a row limit, set it on the Query query.setRange(0, maxParts); } - - LOG.debug("Filter specified is " + filter + "," + - " JDOQL filter is " + queryFilterString); + LOG.debug("Filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString); LOG.debug("Parms is " + params); - String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); query.setResult("partitionName"); - Collection names = (Collection) query.executeWithMap(params); partNames = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { partNames.add((String) i.next()); } - LOG.debug("Done executing query for listMPartitionNamesByFilter"); success = commitTransaction(); LOG.debug("Done retrieving all objects for listMPartitionNamesByFilter"); @@ -2741,6 +2761,9 @@ private String makeParameterDeclarationStringObj(Map params) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return partNames; } @@ -2987,21 +3010,20 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { * @param maxSDs the maximum number of SDs to return * @return a list of storage descriptors */ - private List listStorageDescriptorsWithCD(MColumnDescriptor oldCD, - long maxSDs) { + private List listStorageDescriptorsWithCD(MColumnDescriptor oldCD, long maxSDs) { boolean success = false; + Query query = null; List sds = null; try { openTransaction(); LOG.debug("Executing listStorageDescriptorsWithCD"); - Query query = pm.newQuery(MStorageDescriptor.class, - "this.cd == inCD"); + query = pm.newQuery(MStorageDescriptor.class, "this.cd == inCD"); query.declareParameters("MColumnDescriptor inCD"); - if(maxSDs >= 0) { - //User specified a row limit, set it on the Query + if (maxSDs >= 0) { + // User specified a row limit, set it on the Query query.setRange(0, maxSDs); } - sds = (List) query.execute(oldCD); + sds = cloneQueryResult((List)query.execute(oldCD)); LOG.debug("Done executing query for listStorageDescriptorsWithCD"); pm.retrieveAll(sds); success = commitTransaction(); @@ -3010,6 +3032,9 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return sds; } @@ -3078,9 +3103,11 @@ public boolean dropIndex(String dbName, String origTableName, String indexName) return success; } - private MIndex getMIndex(String dbName, String originalTblName, String indexName) throws MetaException { + private MIndex getMIndex(String dbName, String originalTblName, String indexName) + throws MetaException { MIndex midx = null; boolean commited = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -3090,19 +3117,23 @@ private MIndex getMIndex(String dbName, String originalTblName, String indexName commited = commitTransaction(); return null; } - - Query query = pm.newQuery(MIndex.class, - "origTable.tableName == t1 && origTable.database.name == t2 && indexName == t3"); + query = + pm.newQuery(MIndex.class, + "origTable.tableName == t1 && origTable.database.name == t2 && indexName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); - midx = (MIndex) query.execute(originalTblName, dbName, - HiveStringUtils.normalizeIdentifier(indexName)); + midx = + (MIndex) query.execute(originalTblName, dbName, + HiveStringUtils.normalizeIdentifier(indexName)); pm.retrieve(midx); commited = commitTransaction(); } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return midx; } @@ -3159,19 +3190,19 @@ private Index convertToIndex(MIndex mIndex) throws MetaException { } } - private List listMIndexes(String dbName, String origTableName, - int max) { + private List listMIndexes(String dbName, String origTableName, int max) { boolean success = false; + Query query = null; List mindexes = null; try { openTransaction(); LOG.debug("Executing listMIndexes"); dbName = HiveStringUtils.normalizeIdentifier(dbName); origTableName = HiveStringUtils.normalizeIdentifier(origTableName); - Query query = pm.newQuery(MIndex.class, - "origTable.tableName == t1 && origTable.database.name == t2"); + query = + pm.newQuery(MIndex.class, "origTable.tableName == t1 && origTable.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mindexes = (List) query.execute(origTableName, dbName); + mindexes = cloneQueryResult((List) query.execute(origTableName, dbName)); LOG.debug("Done executing query for listMIndexes"); pm.retrieveAll(mindexes); success = commitTransaction(); @@ -3180,27 +3211,31 @@ private Index convertToIndex(MIndex mIndex) throws MetaException { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mindexes; } @Override - public List listIndexNames(String dbName, String origTableName, - short max) throws MetaException { + public List listIndexNames(String dbName, String origTableName, short max) + throws MetaException { List pns = new ArrayList(); boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listIndexNames"); dbName = HiveStringUtils.normalizeIdentifier(dbName); origTableName = HiveStringUtils.normalizeIdentifier(origTableName); - Query q = pm.newQuery( - "select indexName from org.apache.hadoop.hive.metastore.model.MIndex " - + "where origTable.database.name == t1 && origTable.tableName == t2 " - + "order by indexName asc"); - q.declareParameters("java.lang.String t1, java.lang.String t2"); - q.setResult("indexName"); - Collection names = (Collection) q.execute(dbName, origTableName); + query = + pm.newQuery("select indexName from org.apache.hadoop.hive.metastore.model.MIndex " + + "where origTable.database.name == t1 && origTable.tableName == t2 " + + "order by indexName asc"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.setResult("indexName"); + Collection names = (Collection) query.execute(dbName, origTableName); for (Iterator i = names.iterator(); i.hasNext();) { pns.add((String) i.next()); } @@ -3209,6 +3244,9 @@ private Index convertToIndex(MIndex mIndex) throws MetaException { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return pns; } @@ -3316,13 +3354,16 @@ public boolean revokeRole(Role role, String userName, PrincipalType principalTyp return success; } - private MRoleMap getMSecurityUserRoleMap(String userName, - PrincipalType principalType, String roleName) { + private MRoleMap getMSecurityUserRoleMap(String userName, PrincipalType principalType, + String roleName) { MRoleMap mRoleMember = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2 && role.roleName == t3"); + query = + pm.newQuery(MRoleMap.class, + "principalName == t1 && principalType == t2 && role.roleName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); mRoleMember = (MRoleMap) query.executeWithArray(userName, principalType.toString(), roleName); @@ -3332,6 +3373,9 @@ private MRoleMap getMSecurityUserRoleMap(String userName, if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mRoleMember; } @@ -3443,22 +3487,18 @@ private void getAllRoleAncestors(Set processedRoleNames, List @SuppressWarnings("unchecked") @Override - public List listRoles(String principalName, - PrincipalType principalType) { + public List listRoles(String principalName, PrincipalType principalType) { boolean success = false; + Query query = null; List mRoleMember = null; try { openTransaction(); LOG.debug("Executing listRoles"); - Query query = pm - .newQuery( - MRoleMap.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); + query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); query.setUnique(false); - mRoleMember = (List) query.executeWithArray( - principalName, principalType.toString()); + mRoleMember = + cloneQueryResult((List) query.executeWithArray(principalName, principalType.toString())); LOG.debug("Done executing query for listMSecurityUserRoleMap"); pm.retrieveAll(mRoleMember); success = commitTransaction(); @@ -3467,8 +3507,10 @@ private void getAllRoleAncestors(Set processedRoleNames, List if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - if (principalType == PrincipalType.USER) { // All users belong to public role implicitly, add that role if (mRoleMember == null) { @@ -3477,36 +3519,35 @@ private void getAllRoleAncestors(Set processedRoleNames, List mRoleMember = new ArrayList(mRoleMember); } MRole publicRole = new MRole(HiveMetaStore.PUBLIC, 0, HiveMetaStore.PUBLIC); - mRoleMember.add(new MRoleMap(principalName, principalType.toString(), publicRole, 0, - null, null, false)); + mRoleMember.add(new MRoleMap(principalName, principalType.toString(), publicRole, 0, null, + null, false)); } return mRoleMember; - } @SuppressWarnings("unchecked") private List listMSecurityPrincipalMembershipRole(final String roleName, final PrincipalType principalType) { boolean success = false; + Query query = null; List mRoleMemebership = null; try { openTransaction(); LOG.debug("Executing listMSecurityPrincipalMembershipRole"); - Query query = pm.newQuery(MRoleMap.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mRoleMemebership = (List) query.execute(roleName, principalType.toString()); - LOG - .debug("Done executing query for listMSecurityPrincipalMembershipRole"); + query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mRoleMemebership = cloneQueryResult((List) query.execute(roleName, principalType.toString())); + LOG.debug("Done executing query for listMSecurityPrincipalMembershipRole"); pm.retrieveAll(mRoleMemebership); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listMSecurityPrincipalMembershipRole"); + LOG.debug("Done retrieving all objects for listMSecurityPrincipalMembershipRole"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mRoleMemebership; } @@ -3525,9 +3566,10 @@ public Role getRole(String roleName) throws NoSuchObjectException { private MRole getMRole(String roleName) { MRole mrole = null; boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MRole.class, "roleName == t1"); + query = pm.newQuery(MRole.class, "roleName == t1"); query.declareParameters("java.lang.String t1"); query.setUnique(true); mrole = (MRole) query.execute(roleName); @@ -3537,6 +3579,9 @@ private MRole getMRole(String roleName) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mrole; } @@ -3544,13 +3589,14 @@ private MRole getMRole(String roleName) { @Override public List listRoleNames() { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listAllRoleNames"); - Query query = pm.newQuery("select roleName from org.apache.hadoop.hive.metastore.model.MRole"); + query = pm.newQuery("select roleName from org.apache.hadoop.hive.metastore.model.MRole"); query.setResult("roleName"); Collection names = (Collection) query.execute(); - List roleNames = new ArrayList(); + List roleNames = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { roleNames.add((String) i.next()); } @@ -3560,6 +3606,9 @@ private MRole getMRole(String roleName) { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -4368,16 +4417,15 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) @Override public List listRoleMembers(String roleName) { boolean success = false; + Query query = null; List mRoleMemeberList = null; try { openTransaction(); LOG.debug("Executing listMSecurityUserRoleMember"); - Query query = pm.newQuery(MRoleMap.class, - "role.roleName == t1"); + query = pm.newQuery(MRoleMap.class, "role.roleName == t1"); query.declareParameters("java.lang.String t1"); query.setUnique(false); - mRoleMemeberList = (List) query.execute( - roleName); + mRoleMemeberList = cloneQueryResult((List) query.execute(roleName)); LOG.debug("Done executing query for listMSecurityUserRoleMember"); pm.retrieveAll(mRoleMemeberList); success = commitTransaction(); @@ -4386,24 +4434,28 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mRoleMemeberList; } @SuppressWarnings("unchecked") @Override - public List listPrincipalGlobalGrants(String principalName, PrincipalType principalType) { + public List listPrincipalGlobalGrants(String principalName, + PrincipalType principalType) { boolean commited = false; + Query query = null; List userNameDbPriv = null; try { openTransaction(); if (principalName != null) { - Query query = pm.newQuery(MGlobalPrivilege.class, - "principalName == t1 && principalType == t2 "); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - userNameDbPriv = (List) query - .executeWithArray(principalName, principalType.toString()); + query = pm.newQuery(MGlobalPrivilege.class, "principalName == t1 && principalType == t2 "); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + userNameDbPriv = + cloneQueryResult((List) query + .executeWithArray(principalName, principalType.toString())); pm.retrieveAll(userNameDbPriv); } commited = commitTransaction(); @@ -4411,6 +4463,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return userNameDbPriv; } @@ -4418,10 +4473,11 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) @Override public List listGlobalGrantsAll() { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MGlobalPrivilege.class); - List userNameDbPriv = (List) query.execute(); + query = pm.newQuery(MGlobalPrivilege.class); + List userNameDbPriv = cloneQueryResult((List) query.execute()); pm.retrieveAll(userNameDbPriv); commited = commitTransaction(); return convertGlobal(userNameDbPriv); @@ -4429,6 +4485,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -4452,17 +4511,19 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) public List listPrincipalDBGrants(String principalName, PrincipalType principalType, String dbName) { boolean success = false; + Query query = null; List mSecurityDBList = null; dbName = HiveStringUtils.normalizeIdentifier(dbName); - try { openTransaction(); LOG.debug("Executing listPrincipalDBGrants"); - Query query = pm.newQuery(MDBPrivilege.class, - "principalName == t1 && principalType == t2 && database.name == t3"); - query - .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - mSecurityDBList = (List) query.executeWithArray(principalName, principalType.toString(), dbName); + query = + pm.newQuery(MDBPrivilege.class, + "principalName == t1 && principalType == t2 && database.name == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + mSecurityDBList = + cloneQueryResult((List) query.executeWithArray(principalName, principalType.toString(), + dbName)); LOG.debug("Done executing query for listPrincipalDBGrants"); pm.retrieveAll(mSecurityDBList); success = commitTransaction(); @@ -4471,6 +4532,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityDBList; } @@ -4504,22 +4568,22 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) } @SuppressWarnings("unchecked") - private List listPrincipalAllDBGrant( - String principalName, PrincipalType principalType) { + private List listPrincipalAllDBGrant(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; List mSecurityDBList = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllDBGrant"); if (principalName != null && principalType != null) { - Query query = pm.newQuery(MDBPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityDBList = (List) query.execute(principalName, principalType.toString()); + query = pm.newQuery(MDBPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityDBList = + cloneQueryResult((List) query.execute(principalName, principalType.toString())); } else { - Query query = pm.newQuery(MDBPrivilege.class); - mSecurityDBList = (List) query.execute(); + query = pm.newQuery(MDBPrivilege.class); + mSecurityDBList = cloneQueryResult((List) query.execute()); } LOG.debug("Done executing query for listPrincipalAllDBGrant"); pm.retrieveAll(mSecurityDBList); @@ -4529,14 +4593,17 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityDBList; } @SuppressWarnings("unchecked") - public List listAllTableGrants(String dbName, - String tableName) { + public List listAllTableGrants(String dbName, String tableName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); List mSecurityTabList = null; @@ -4546,71 +4613,68 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) openTransaction(); LOG.debug("Executing listAllTableGrants"); String queryStr = "table.tableName == t1 && table.database.name == t2"; - Query query = pm.newQuery( - MTablePrivilege.class, queryStr); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - mSecurityTabList = (List) query - .executeWithArray(tableName, dbName); + query = pm.newQuery(MTablePrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityTabList = cloneQueryResult((List) query.executeWithArray(tableName, dbName)); LOG.debug("Done executing query for listAllTableGrants"); pm.retrieveAll(mSecurityTabList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listAllTableGrants"); + LOG.debug("Done retrieving all objects for listAllTableGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabList; } @SuppressWarnings("unchecked") - public List listTableAllPartitionGrants(String dbName, - String tableName) { + public List listTableAllPartitionGrants(String dbName, String tableName) { tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); boolean success = false; + Query query = null; List mSecurityTabPartList = null; try { openTransaction(); LOG.debug("Executing listTableAllPartitionGrants"); String queryStr = "partition.table.tableName == t1 && partition.table.database.name == t2"; - Query query = pm.newQuery( - MPartitionPrivilege.class, queryStr); - query.declareParameters( - "java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query - .executeWithArray(tableName, dbName); + query = pm.newQuery(MPartitionPrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityTabPartList = cloneQueryResult((List) query.executeWithArray(tableName, dbName)); LOG.debug("Done executing query for listTableAllPartitionGrants"); pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listTableAllPartitionGrants"); + LOG.debug("Done retrieving all objects for listTableAllPartitionGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @SuppressWarnings("unchecked") - public List listTableAllColumnGrants(String dbName, - String tableName) { + public List listTableAllColumnGrants(String dbName, String tableName) { boolean success = false; + Query query = null; List mTblColPrivilegeList = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - try { openTransaction(); LOG.debug("Executing listTableAllColumnGrants"); String queryStr = "table.tableName == t1 && table.database.name == t2"; - Query query = pm.newQuery(MTableColumnPrivilege.class, queryStr); + query = pm.newQuery(MTableColumnPrivilege.class, queryStr); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mTblColPrivilegeList = (List) query - .executeWithArray(tableName, dbName); + mTblColPrivilegeList = + cloneQueryResult((List) query.executeWithArray(tableName, dbName)); LOG.debug("Done executing query for listTableAllColumnGrants"); pm.retrieveAll(mTblColPrivilegeList); success = commitTransaction(); @@ -4619,6 +4683,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mTblColPrivilegeList; } @@ -4627,18 +4694,18 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) public List listTableAllPartitionColumnGrants(String dbName, String tableName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - List mSecurityColList = null; try { openTransaction(); LOG.debug("Executing listTableAllPartitionColumnGrants"); String queryStr = "partition.table.tableName == t1 && partition.table.database.name == t2"; - Query query = pm.newQuery(MPartitionColumnPrivilege.class, queryStr); + query = pm.newQuery(MPartitionColumnPrivilege.class, queryStr); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColList = (List) query - .executeWithArray(tableName, dbName); + mSecurityColList = + cloneQueryResult((List) query.executeWithArray(tableName, dbName)); LOG.debug("Done executing query for listTableAllPartitionColumnGrants"); pm.retrieveAll(mSecurityColList); success = commitTransaction(); @@ -4647,6 +4714,9 @@ public boolean revokePrivileges(PrivilegeBag privileges, boolean grantOption) if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @@ -4688,16 +4758,14 @@ public void dropPartitionAllColumnGrantsNoTxn( @SuppressWarnings("unchecked") private List listDatabaseGrants(String dbName) { dbName = HiveStringUtils.normalizeIdentifier(dbName); - boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listDatabaseGrants"); - Query query = pm.newQuery(MDBPrivilege.class, - "database.name == t1"); + query = pm.newQuery(MDBPrivilege.class, "database.name == t1"); query.declareParameters("java.lang.String t1"); - List mSecurityDBList = (List) query - .executeWithArray(dbName); + List mSecurityDBList = cloneQueryResult((List) query.executeWithArray(dbName)); LOG.debug("Done executing query for listDatabaseGrants"); pm.retrieveAll(mSecurityDBList); success = commitTransaction(); @@ -4707,6 +4775,9 @@ public void dropPartitionAllColumnGrantsNoTxn( if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -4774,61 +4845,62 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listAllTableGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName) { + public List listAllTableGrants(String principalName, + PrincipalType principalType, String dbName, String tableName) { tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - boolean success = false; + Query query = null; List mSecurityTabPartList = null; try { openTransaction(); LOG.debug("Executing listAllTableGrants"); - Query query = pm.newQuery( - MTablePrivilege.class, + query = + pm.newQuery(MTablePrivilege.class, "principalName == t1 && principalType == t2 && table.tableName == t3 && table.database.name == t4"); - query.declareParameters( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); - mSecurityTabPartList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, dbName); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); + mSecurityTabPartList = + cloneQueryResult((List) query.executeWithArray(principalName, principalType.toString(), + tableName, dbName)); LOG.debug("Done executing query for listAllTableGrants"); pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listAllTableGrants"); + LOG.debug("Done retrieving all objects for listAllTableGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @SuppressWarnings("unchecked") @Override - public List listPrincipalPartitionGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String partName) { + public List listPrincipalPartitionGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); - List mSecurityTabPartList = null; try { openTransaction(); LOG.debug("Executing listMSecurityPrincipalPartitionGrant"); - Query query = pm.newQuery( - MPartitionPrivilege.class, - "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + - "&& partition.table.database.name == t4 && partition.partitionName == t5"); - query.declareParameters( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4, " + - "java.lang.String t5"); - mSecurityTabPartList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, dbName, partName); + query = + pm.newQuery(MPartitionPrivilege.class, + "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + + "&& partition.table.database.name == t4 && partition.partitionName == t5"); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4, " + + "java.lang.String t5"); + mSecurityTabPartList = + cloneQueryResult((List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, partName)); LOG.debug("Done executing query for listMSecurityPrincipalPartitionGrant"); - pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); LOG.debug("Done retrieving all objects for listMSecurityPrincipalPartitionGrant"); @@ -4836,16 +4908,19 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalTableColumnGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String columnName) { + public List listPrincipalTableColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String columnName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); columnName = HiveStringUtils.normalizeIdentifier(columnName); @@ -4853,83 +4928,88 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List) query.executeWithArray( - principalName, principalType.toString(), tableName, dbName, columnName); + String queryStr = + "principalName == t1 && principalType == t2 && " + + "table.tableName == t3 && table.database.name == t4 && columnName == t5 "; + query = pm.newQuery(MTableColumnPrivilege.class, queryStr); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + + "java.lang.String t4, java.lang.String t5"); + mSecurityColList = + cloneQueryResult((List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, columnName)); LOG.debug("Done executing query for listPrincipalTableColumnGrants"); pm.retrieveAll(mSecurityColList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalTableColumnGrants"); + LOG.debug("Done retrieving all objects for listPrincipalTableColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @Override @SuppressWarnings("unchecked") - public List listPrincipalPartitionColumnGrants( - String principalName, PrincipalType principalType, String dbName, - String tableName, String partitionName, String columnName) { + public List listPrincipalPartitionColumnGrants(String principalName, + PrincipalType principalType, String dbName, String tableName, String partitionName, + String columnName) { boolean success = false; + Query query = null; tableName = HiveStringUtils.normalizeIdentifier(tableName); dbName = HiveStringUtils.normalizeIdentifier(dbName); columnName = HiveStringUtils.normalizeIdentifier(columnName); - List mSecurityColList = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionColumnGrants"); - Query query = pm - .newQuery( + query = + pm.newQuery( MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + - "&& partition.table.database.name == t4 && partition.partitionName == t5 && columnName == t6"); - query - .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + - "java.lang.String t4, java.lang.String t5, java.lang.String t6"); - - mSecurityColList = (List) query - .executeWithArray(principalName, principalType.toString(), tableName, - dbName, partitionName, columnName); + "principalName == t1 && principalType == t2 && partition.table.tableName == t3 " + + "&& partition.table.database.name == t4 && partition.partitionName == t5 && columnName == t6"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + + "java.lang.String t4, java.lang.String t5, java.lang.String t6"); + mSecurityColList = + cloneQueryResult((List) query.executeWithArray(principalName, + principalType.toString(), tableName, dbName, partitionName, columnName)); LOG.debug("Done executing query for listPrincipalPartitionColumnGrants"); pm.retrieveAll(mSecurityColList); - success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalPartitionColumnGrants"); + LOG.debug("Done retrieving all objects for listPrincipalPartitionColumnGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityColList; } @Override - public List listPrincipalPartitionColumnGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalPartitionColumnGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionColumnGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MPartitionColumnPrivilege.class, + "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.executeWithArray(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.executeWithArray(principalName, + principalType.toString()); } else { - Query query = pm.newQuery(MPartitionColumnPrivilege.class); + query = pm.newQuery(MPartitionColumnPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalPartitionColumnGrantsAll"); @@ -4942,23 +5022,29 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPartitionColumnGrantsAll( - String dbName, String tableName, String partitionName, String columnName) { + public List listPartitionColumnGrantsAll(String dbName, String tableName, + String partitionName, String columnName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPartitionColumnGrantsAll"); - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "partition.table.tableName == t3 && partition.table.database.name == t4 && " + - "partition.partitionName == t5 && columnName == t6"); - query.declareParameters( - "java.lang.String t3, java.lang.String t4, java.lang.String t5, java.lang.String t6"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, partitionName, columnName); + query = + pm.newQuery(MPartitionColumnPrivilege.class, + "partition.table.tableName == t3 && partition.table.database.name == t4 && " + + "partition.partitionName == t5 && columnName == t6"); + query + .declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5, java.lang.String t6"); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, + partitionName, columnName); LOG.debug("Done executing query for listPartitionColumnGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertPartCols(mSecurityTabPartList); @@ -4969,6 +5055,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalTableGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalTableGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllTableGrants"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MTablePrivilege.class, - "principalName == t1 && principalType == t2"); + query = pm.newQuery(MTablePrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query.execute( - principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MTablePrivilege.class); + query = pm.newQuery(MTablePrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalAllTableGrants"); @@ -5047,20 +5136,24 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listTableGrantsAll(String dbName, String tableName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listTableGrantsAll"); - Query query = pm.newQuery(MTablePrivilege.class, - "table.tableName == t1 && table.database.name == t2"); + query = + pm.newQuery(MTablePrivilege.class, "table.tableName == t1 && table.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName); LOG.debug("Done executing query for listTableGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertTable(mSecurityTabPartList); @@ -5071,6 +5164,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllPartitionGrants( - String principalName, PrincipalType principalType) { + private List listPrincipalAllPartitionGrants(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; List mSecurityTabPartList = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllPartitionGrants"); - Query query = pm.newQuery(MPartitionPrivilege.class, - "principalName == t1 && principalType == t2"); + query = pm.newQuery(MPartitionPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) query.execute( - principalName, principalType.toString()); - LOG - .debug("Done executing query for listPrincipalAllPartitionGrants"); + mSecurityTabPartList = + cloneQueryResult((List) query.execute(principalName, principalType.toString())); + LOG.debug("Done executing query for listPrincipalAllPartitionGrants"); pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); - LOG - .debug("Done retrieving all objects for listPrincipalAllPartitionGrants"); + LOG.debug("Done retrieving all objects for listPrincipalAllPartitionGrants"); } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mSecurityTabPartList; } @Override - public List listPrincipalPartitionGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalPartitionGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MPartitionPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MPartitionPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.execute(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MPartitionPrivilege.class); + query = pm.newQuery(MPartitionPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalPartitionGrantsAll"); @@ -5148,22 +5246,27 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPartitionGrantsAll( - String dbName, String tableName, String partitionName) { + public List listPartitionGrantsAll(String dbName, String tableName, + String partitionName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalPartitionGrantsAll"); - Query query = pm.newQuery(MPartitionPrivilege.class, - "partition.table.tableName == t3 && partition.table.database.name == t4 && " + - "partition.partitionName == t5"); + query = + pm.newQuery(MPartitionPrivilege.class, + "partition.table.tableName == t3 && partition.table.database.name == t4 && " + + "partition.partitionName == t5"); query.declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, partitionName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, partitionName); LOG.debug("Done executing query for listPrincipalPartitionGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertPartition(mSecurityTabPartList); @@ -5174,6 +5277,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllTableColumnGrants( - String principalName, PrincipalType principalType) { + private List listPrincipalAllTableColumnGrants(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; List mSecurityColumnList = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllTableColumnGrants"); - Query query = pm.newQuery(MTableColumnPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColumnList = (List) query.execute( - principalName, principalType.toString()); + query = + pm.newQuery(MTableColumnPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityColumnList = + cloneQueryResult((List) query.execute(principalName, principalType.toString())); LOG.debug("Done executing query for listPrincipalAllTableColumnGrants"); pm.retrieveAll(mSecurityColumnList); success = commitTransaction(); @@ -5219,27 +5325,31 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalTableColumnGrantsAll( - String principalName, PrincipalType principalType) { + public List listPrincipalTableColumnGrantsAll(String principalName, + PrincipalType principalType) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalTableColumnGrantsAll"); List mSecurityTabPartList; if (principalName != null && principalType != null) { - Query query = pm.newQuery(MTableColumnPrivilege.class, - "principalName == t1 && principalType == t2"); + query = + pm.newQuery(MTableColumnPrivilege.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityTabPartList = (List) - query.execute(principalName, principalType.toString()); + mSecurityTabPartList = + (List) query.execute(principalName, principalType.toString()); } else { - Query query = pm.newQuery(MTableColumnPrivilege.class); + query = pm.newQuery(MTableColumnPrivilege.class); mSecurityTabPartList = (List) query.execute(); } LOG.debug("Done executing query for listPrincipalTableColumnGrantsAll"); @@ -5252,21 +5362,26 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listTableColumnGrantsAll( - String dbName, String tableName, String columnName) { + public List listTableColumnGrantsAll(String dbName, String tableName, + String columnName) { boolean success = false; + Query query = null; try { openTransaction(); LOG.debug("Executing listPrincipalTableColumnGrantsAll"); - Query query = pm.newQuery(MTableColumnPrivilege.class, - "table.tableName == t3 && table.database.name == t4 && columnName == t5"); + query = + pm.newQuery(MTableColumnPrivilege.class, + "table.tableName == t3 && table.database.name == t4 && columnName == t5"); query.declareParameters("java.lang.String t3, java.lang.String t4, java.lang.String t5"); - List mSecurityTabPartList = (List) - query.executeWithArray(tableName, dbName, columnName); + List mSecurityTabPartList = + (List) query.executeWithArray(tableName, dbName, columnName); LOG.debug("Done executing query for listPrincipalTableColumnGrantsAll"); pm.retrieveAll(mSecurityTabPartList); List result = convertTableCols(mSecurityTabPartList); @@ -5277,6 +5392,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List listPrincipalAllPartitionColumnGrants( String principalName, PrincipalType principalType) { boolean success = false; + Query query = null; List mSecurityColumnList = null; try { openTransaction(); LOG.debug("Executing listPrincipalAllTableColumnGrants"); - Query query = pm.newQuery(MPartitionColumnPrivilege.class, - "principalName == t1 && principalType == t2"); - query - .declareParameters("java.lang.String t1, java.lang.String t2"); - mSecurityColumnList = (List) query.execute( - principalName, principalType.toString()); + query = + pm.newQuery(MPartitionColumnPrivilege.class, "principalName == t1 && principalType == t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2"); + mSecurityColumnList = + cloneQueryResult((List) query.execute(principalName, principalType.toString())); LOG.debug("Done executing query for listPrincipalAllTableColumnGrants"); pm.retrieveAll(mSecurityColumnList); success = commitTransaction(); @@ -5321,6 +5439,9 @@ private void dropPartitionGrantsNoTxn(String dbName, String tableName, List partName, PartitionEventType evtType) throws UnknownTableException, MetaException, InvalidPartitionException, UnknownPartitionException { - Collection partEvents; boolean success = false; + Query query = null; LOG.debug("Begin Executing isPartitionMarkedForEvent"); - try{ - openTransaction(); - Query query = pm.newQuery(MPartitionEvent.class, "dbName == t1 && tblName == t2 && partName == t3 && eventType == t4"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4"); - Table tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. - if(null == tbl) { - throw new UnknownTableException("Table: "+ tblName + " is not found."); - } - partEvents = (Collection) query.executeWithArray(dbName, tblName, getPartitionStr(tbl, partName), evtType.getValue()); - pm.retrieveAll(partEvents); - success = commitTransaction(); - LOG.debug("Done executing isPartitionMarkedForEvent"); - } finally{ + try { + openTransaction(); + query = + pm.newQuery(MPartitionEvent.class, + "dbName == t1 && tblName == t2 && partName == t3 && eventType == t4"); + query + .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4"); + Table tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. + if (null == tbl) { + throw new UnknownTableException("Table: " + tblName + " is not found."); + } + partEvents = + cloneQueryResult((Collection) query.executeWithArray(dbName, tblName, + getPartitionStr(tbl, partName), evtType.getValue())); + pm.retrieveAll(partEvents); + success = commitTransaction(); + LOG.debug("Done executing isPartitionMarkedForEvent"); + } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - return (partEvents != null && !partEvents.isEmpty()) ? true : false; - + return (partEvents != null && !partEvents.isEmpty()) ? true : false; } @Override @@ -5401,14 +5529,14 @@ private String getPartitionStr(Table tbl, Map partName) throws In * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public Collection executeJDOQLSelect(String query) { + public Collection executeJDOQLSelect(String queryStr) { boolean committed = false; Collection result = null; - + Query query = null; try { openTransaction(); - Query q = pm.newQuery(query); - result = (Collection) q.execute(); + query = pm.newQuery(queryStr); + result = cloneQueryResult((Collection) query.execute()); committed = commitTransaction(); if (committed) { return result; @@ -5419,6 +5547,9 @@ private String getPartitionStr(Table tbl, Map partName) throws In if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5429,14 +5560,14 @@ private String getPartitionStr(Table tbl, Map partName) throws In * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public long executeJDOQLUpdate(String query) { + public long executeJDOQLUpdate(String queryStr) { boolean committed = false; long numUpdated = 0; - + Query query = null; try { openTransaction(); - Query q = pm.newQuery(query); - numUpdated = (Long) q.execute(); + query = pm.newQuery(queryStr); + numUpdated = (Long) query.execute(); committed = commitTransaction(); if (committed) { return numUpdated; @@ -5447,6 +5578,9 @@ public long executeJDOQLUpdate(String query) { if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5459,15 +5593,14 @@ public long executeJDOQLUpdate(String query) { */ public Set listFSRoots() { boolean committed = false; + Query query = null; Set fsRoots = new HashSet(); - try { openTransaction(); - Query query = pm.newQuery(MDatabase.class); + query = pm.newQuery(MDatabase.class); List mDBs = (List) query.execute(); pm.retrieveAll(mDBs); - - for (MDatabase mDB:mDBs) { + for (MDatabase mDB : mDBs) { fsRoots.add(mDB.getLocationUri()); } committed = commitTransaction(); @@ -5480,6 +5613,9 @@ public long executeJDOQLUpdate(String query) { if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5554,22 +5690,22 @@ public void setUpdateLocations(Map updateLocations) { */ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boolean dryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateMDatabaseURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MDatabase.class); + query = pm.newQuery(MDatabase.class); List mDBs = (List) query.execute(); pm.retrieveAll(mDBs); - for(MDatabase mDB:mDBs) { + for (MDatabase mDB : mDBs) { URI locationURI = null; String location = mDB.getLocationUri(); try { locationURI = new URI(location); - } catch(URISyntaxException e) { + } catch (URISyntaxException e) { badRecords.add(location); } catch (NullPointerException e) { badRecords.add(location); @@ -5595,6 +5731,9 @@ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boole if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5661,16 +5800,16 @@ private void updatePropURIHelper(URI oldLoc, URI newLoc, String tblPropKey, bool * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, boolean - isDryRun) { + public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPropKey, + boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap<>(); List badRecords = new ArrayList<>(); UpdatePropURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MTable.class); + query = pm.newQuery(MTable.class); List mTbls = (List) query.execute(); pm.retrieveAll(mTbls); @@ -5687,6 +5826,9 @@ public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPr if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5698,20 +5840,19 @@ public UpdatePropURIRetVal updateTblPropURI(URI oldLoc, URI newLoc, String tblPr * */ @Deprecated - public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, - URI newLoc, String tblPropKey, boolean isDryRun) { + public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, URI newLoc, + String tblPropKey, boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdatePropURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MStorageDescriptor.class); + query = pm.newQuery(MStorageDescriptor.class); List mSDSs = (List) query.execute(); pm.retrieveAll(mSDSs); - - for(MStorageDescriptor mSDS:mSDSs) { + for (MStorageDescriptor mSDS : mSDSs) { updatePropURIHelper(oldLoc, newLoc, tblPropKey, isDryRun, badRecords, updateLocations, mSDS.getParameters()); } @@ -5720,11 +5861,14 @@ public UpdatePropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc, retVal = new UpdatePropURIRetVal(badRecords, updateLocations); } return retVal; - } finally { - if (!committed) { - rollbackTransaction(); - } - } + } finally { + if (!committed) { + rollbackTransaction(); + } + if (query != null) { + query.closeAll(); + } + } } public class UpdateMStorageDescriptorTblURIRetVal { @@ -5761,20 +5905,19 @@ public void setUpdateLocations(Map updateLocations) { * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift. * */ - public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, URI newLoc, - boolean isDryRun) { + public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, + URI newLoc, boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateMStorageDescriptorTblURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MStorageDescriptor.class); + query = pm.newQuery(MStorageDescriptor.class); List mSDSs = (List) query.execute(); pm.retrieveAll(mSDSs); - - for(MStorageDescriptor mSDS:mSDSs) { + for (MStorageDescriptor mSDS : mSDSs) { URI locationURI = null; String location = mSDS.getLocation(); try { @@ -5802,10 +5945,13 @@ public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI o } return retVal; } finally { - if (!committed) { - rollbackTransaction(); - } - } + if (!committed) { + rollbackTransaction(); + } + if (query != null) { + query.closeAll(); + } + } } public class UpdateSerdeURIRetVal { @@ -5842,19 +5988,18 @@ public void setUpdateLocations(Map updateLocations) { * */ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeProp, - boolean isDryRun) { + boolean isDryRun) { boolean committed = false; + Query query = null; Map updateLocations = new HashMap(); List badRecords = new ArrayList(); UpdateSerdeURIRetVal retVal = null; - try { openTransaction(); - Query query = pm.newQuery(MSerDeInfo.class); + query = pm.newQuery(MSerDeInfo.class); List mSerdes = (List) query.execute(); pm.retrieveAll(mSerdes); - - for(MSerDeInfo mSerde:mSerdes) { + for (MSerDeInfo mSerde : mSerdes) { if (mSerde.getParameters().containsKey(serdeProp)) { String schemaLoc = mSerde.getParameters().get(serdeProp); URI schemaLocURI = null; @@ -5887,6 +6032,9 @@ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeP if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -5946,7 +6094,7 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, pm.makePersistent(mStatsObj); } } - + @Override public boolean updateTableColumnStatistics(ColumnStatistics colStats) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { @@ -6007,15 +6155,15 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List getMTableColumnStatistics( - Table table, List colNames) throws MetaException { + private List getMTableColumnStatistics(Table table, List colNames) + throws MetaException { boolean committed = false; + Query query = null; openTransaction(); try { List result = null; validateTableCols(table, colNames); - - Query query = pm.newQuery(MTableColumnStatistics.class); + query = pm.newQuery(MTableColumnStatistics.class); String filter = "tableName == t1 && dbName == t2 && ("; String paramStr = "java.lang.String t1, java.lang.String t2"; Object[] params = new Object[colNames.size() + 2]; @@ -6029,18 +6177,18 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List) query.executeWithArray(params); + result = cloneQueryResult((List) query.executeWithArray(params)); pm.retrieveAll(result); if (result.size() > colNames.size()) { - throw new MetaException( - "Unexpected " + result.size() + " statistics for " + colNames.size() + " columns"); + throw new MetaException("Unexpected " + result.size() + " statistics for " + + colNames.size() + " columns"); } committed = commitTransaction(); return result; } catch (Exception ex) { LOG.error("Error retrieving statistics via jdo", ex); if (ex instanceof MetaException) { - throw (MetaException)ex; + throw (MetaException) ex; } throw new MetaException(ex.getMessage()); } finally { @@ -6048,6 +6196,9 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List getMPartitionColumnStatistics( - Table table, List partNames, List colNames) - throws NoSuchObjectException, MetaException { + private List getMPartitionColumnStatistics(Table table, + List partNames, List colNames) throws NoSuchObjectException, MetaException { boolean committed = false; + Query query = null; MPartitionColumnStatistics mStatsObj = null; try { openTransaction(); // We are not going to verify SD for each partition. Just verify for the table. validateTableCols(table, colNames); boolean foundCol = false; - Query query = pm.newQuery(MPartitionColumnStatistics.class); + query = pm.newQuery(MPartitionColumnStatistics.class); String paramStr = "java.lang.String t1, java.lang.String t2"; String filter = "tableName == t1 && dbName == t2 && ("; Object[] params = new Object[colNames.size() + partNames.size() + 2]; @@ -6215,14 +6366,14 @@ protected String describeResult() { query.setOrdering("partitionName ascending"); @SuppressWarnings("unchecked") List result = - (List) query.executeWithArray(params); + cloneQueryResult((List) query.executeWithArray(params)); pm.retrieveAll(result); committed = commitTransaction(); return result; } catch (Exception ex) { LOG.error("Error retrieving statistics via jdo", ex); if (ex instanceof MetaException) { - throw (MetaException)ex; + throw (MetaException) ex; } throw new MetaException(ex.getMessage()); } finally { @@ -6230,6 +6381,9 @@ protected String describeResult() { rollbackTransaction(); return Lists.newArrayList(); } + if (query != null) { + query.closeAll(); + } } } @@ -6242,127 +6396,112 @@ private void dropPartitionColumnStatisticsNoTxn( } @Override - public boolean deletePartitionColumnStatistics(String dbName, String tableName, - String partName, List partVals, String colName) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName, + List partVals, String colName) throws NoSuchObjectException, MetaException, + InvalidObjectException, InvalidInputException { boolean ret = false; - + Query query = null; if (dbName == null) { dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; } - if (tableName == null) { throw new InvalidInputException("Table name is null."); } - try { openTransaction(); MTable mTable = getMTable(dbName, tableName); MPartitionColumnStatistics mStatsObj; List mStatsObjColl; - if (mTable == null) { - throw new - NoSuchObjectException("Table " + tableName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Table " + tableName + + " for which stats deletion is requested doesn't exist"); } - - MPartition mPartition = - getMPartition(dbName, tableName, partVals); - + MPartition mPartition = getMPartition(dbName, tableName, partVals); if (mPartition == null) { - throw new - NoSuchObjectException("Partition " + partName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Partition " + partName + + " for which stats deletion is requested doesn't exist"); } - - Query query = pm.newQuery(MPartitionColumnStatistics.class); + query = pm.newQuery(MPartitionColumnStatistics.class); String filter; String parameters; - if (colName != null) { - filter = "partition.partitionName == t1 && dbName == t2 && tableName == t3 && " + - "colName == t4"; - parameters = "java.lang.String t1, java.lang.String t2, " + - "java.lang.String t3, java.lang.String t4"; + filter = + "partition.partitionName == t1 && dbName == t2 && tableName == t3 && " + + "colName == t4"; + parameters = + "java.lang.String t1, java.lang.String t2, " + + "java.lang.String t3, java.lang.String t4"; } else { filter = "partition.partitionName == t1 && dbName == t2 && tableName == t3"; parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; } - query.setFilter(filter); - query - .declareParameters(parameters); - + query.declareParameters(parameters); if (colName != null) { query.setUnique(true); - mStatsObj = (MPartitionColumnStatistics)query.executeWithArray(partName.trim(), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(colName)); + mStatsObj = + (MPartitionColumnStatistics) query.executeWithArray(partName.trim(), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(colName)); pm.retrieve(mStatsObj); - if (mStatsObj != null) { pm.deletePersistent(mStatsObj); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" +dbName + " table=" + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName + " partition=" + partName + " col=" + colName); } } else { - mStatsObjColl= (List)query.execute(partName.trim(), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(tableName)); + mStatsObjColl = + (List) query.execute(partName.trim(), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(tableName)); pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { pm.deletePersistentAll(mStatsObjColl); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + - " table=" + tableName + " partition" + partName); + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + + tableName + " partition" + partName); } } ret = commitTransaction(); - } catch(NoSuchObjectException e) { - rollbackTransaction(); - throw e; + } catch (NoSuchObjectException e) { + rollbackTransaction(); + throw e; } finally { if (!ret) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return ret; } @Override public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException - { + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { boolean ret = false; - + Query query = null; if (dbName == null) { dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; } - if (tableName == null) { throw new InvalidInputException("Table name is null."); } - try { openTransaction(); MTable mTable = getMTable(dbName, tableName); MTableColumnStatistics mStatsObj; - List mStatsObjColl; - + List mStatsObjColl; if (mTable == null) { - throw new - NoSuchObjectException("Table " + tableName + - " for which stats deletion is requested doesn't exist"); + throw new NoSuchObjectException("Table " + tableName + + " for which stats deletion is requested doesn't exist"); } - - Query query = pm.newQuery(MTableColumnStatistics.class); + query = pm.newQuery(MTableColumnStatistics.class); String filter; String parameters; - if (colName != null) { filter = "table.tableName == t1 && dbName == t2 && colName == t3"; parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; @@ -6372,44 +6511,45 @@ public boolean deleteTableColumnStatistics(String dbName, String tableName, Stri } query.setFilter(filter); - query - .declareParameters(parameters); - + query.declareParameters(parameters); if (colName != null) { query.setUnique(true); - mStatsObj = (MTableColumnStatistics)query.execute( - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(dbName), - HiveStringUtils.normalizeIdentifier(colName)); + mStatsObj = + (MTableColumnStatistics) query.execute(HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(dbName), + HiveStringUtils.normalizeIdentifier(colName)); pm.retrieve(mStatsObj); if (mStatsObj != null) { pm.deletePersistent(mStatsObj); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" +dbName + " table=" + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + tableName + " col=" + colName); } } else { - mStatsObjColl= (List)query.execute( - HiveStringUtils.normalizeIdentifier(tableName), - HiveStringUtils.normalizeIdentifier(dbName)); + mStatsObjColl = + (List) query.execute( + HiveStringUtils.normalizeIdentifier(tableName), + HiveStringUtils.normalizeIdentifier(dbName)); pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { pm.deletePersistentAll(mStatsObjColl); } else { - throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + - " table=" + tableName); + throw new NoSuchObjectException("Column stats doesn't exist for db=" + dbName + " table=" + + tableName); } } ret = commitTransaction(); - } catch(NoSuchObjectException e) { - rollbackTransaction(); - throw e; + } catch (NoSuchObjectException e) { + rollbackTransaction(); + throw e; } finally { if (!ret) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return ret; } @@ -6417,21 +6557,26 @@ public boolean deleteTableColumnStatistics(String dbName, String tableName, Stri @Override public long cleanupEvents() { boolean commited = false; + Query query = null; long delCnt; LOG.debug("Begin executing cleanupEvents"); - Long expiryTime = HiveConf.getTimeVar(getConf(), ConfVars.METASTORE_EVENT_EXPIRY_DURATION, TimeUnit.MILLISECONDS); + Long expiryTime = + HiveConf.getTimeVar(getConf(), ConfVars.METASTORE_EVENT_EXPIRY_DURATION, + TimeUnit.MILLISECONDS); Long curTime = System.currentTimeMillis(); try { openTransaction(); - Query query = pm.newQuery(MPartitionEvent.class,"curTime - eventTime > expiryTime"); + query = pm.newQuery(MPartitionEvent.class, "curTime - eventTime > expiryTime"); query.declareParameters("java.lang.Long curTime, java.lang.Long expiryTime"); delCnt = query.deletePersistentAll(curTime, expiryTime); commited = commitTransaction(); - } - finally { + } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } LOG.debug("Done executing cleanupEvents"); } return delCnt; @@ -6441,7 +6586,11 @@ private MDelegationToken getTokenFrom(String tokenId) { Query query = pm.newQuery(MDelegationToken.class, "tokenIdentifier == tokenId"); query.declareParameters("java.lang.String tokenId"); query.setUnique(true); - return (MDelegationToken)query.execute(tokenId); + MDelegationToken delegationToken = (MDelegationToken) query.execute(tokenId); + if (query != null) { + query.closeAll(); + } + return delegationToken; } @Override @@ -6513,28 +6662,31 @@ public String getToken(String tokenId) { @Override public List getAllTokenIdentifiers() { - LOG.debug("Begin executing getAllTokenIdentifiers"); boolean committed = false; - List tokens; - try{ + Query query = null; + List tokenIdents = new ArrayList(); + + try { openTransaction(); - Query query = pm.newQuery(MDelegationToken.class); - tokens = (List) query.execute(); + query = pm.newQuery(MDelegationToken.class); + List tokens = (List) query.execute(); pm.retrieveAll(tokens); committed = commitTransaction(); + + for (MDelegationToken token : tokens) { + tokenIdents.add(token.getTokenIdentifier()); + } + return tokenIdents; } finally { - if(!committed) { + LOG.debug("Done executing getAllTokenIdentifers with status : " + committed); + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - LOG.debug("Done executing getAllTokenIdentifers with status : " + committed); - List tokenIdents = new ArrayList(tokens.size()); - - for (MDelegationToken token : tokens) { - tokenIdents.add(token.getTokenIdentifier()); - } - return tokenIdents; } @Override @@ -6563,21 +6715,25 @@ public int addMasterKey(String key) throws MetaException{ public void updateMasterKey(Integer id, String key) throws NoSuchObjectException, MetaException { LOG.debug("Begin executing updateMasterKey"); boolean committed = false; + Query query = null; MMasterKey masterKey; - try{ - openTransaction(); - Query query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey)query.execute(id); - if (null != masterKey) { - masterKey.setMasterKey(key); - } - committed = commitTransaction(); + try { + openTransaction(); + query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + masterKey.setMasterKey(key); + } + committed = commitTransaction(); } finally { - if(!committed) { + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } LOG.debug("Done executing updateMasterKey with status : " + committed); if (null == masterKey) { @@ -6592,21 +6748,25 @@ public void updateMasterKey(Integer id, String key) throws NoSuchObjectException public boolean removeMasterKey(Integer id) { LOG.debug("Begin executing removeMasterKey"); boolean success = false; + Query query = null; MMasterKey masterKey; - try{ - openTransaction(); - Query query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey)query.execute(id); - if (null != masterKey) { - pm.deletePersistent(masterKey); - } - success = commitTransaction(); + try { + openTransaction(); + query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + pm.deletePersistent(masterKey); + } + success = commitTransaction(); } finally { - if(!success) { + if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } LOG.debug("Done executing removeMasterKey with status : " + success); return (null != masterKey) && success; @@ -6616,25 +6776,29 @@ public boolean removeMasterKey(Integer id) { public String[] getMasterKeys() { LOG.debug("Begin executing getMasterKeys"); boolean committed = false; + Query query = null; List keys; - try{ + try { openTransaction(); - Query query = pm.newQuery(MMasterKey.class); + query = pm.newQuery(MMasterKey.class); keys = (List) query.execute(); pm.retrieveAll(keys); committed = commitTransaction(); + + String[] masterKeys = new String[keys.size()]; + for (int i = 0; i < keys.size(); i++) { + masterKeys[i] = keys.get(i).getMasterKey(); + } + return masterKeys; } finally { - if(!committed) { + LOG.debug("Done executing getMasterKeys with status : " + committed); + if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } - LOG.debug("Done executing getMasterKeys with status : " + committed); - String[] masterKeys = new String[keys.size()]; - - for (int i = 0; i < keys.size(); i++) { - masterKeys[i] = keys.get(i).getMasterKey(); - } - return masterKeys; } // compare hive version and metastore version @@ -6710,22 +6874,20 @@ public String getMetaStoreSchemaVersion() throws MetaException { } @SuppressWarnings("unchecked") - private MVersionTable getMSchemaVersion() - throws NoSuchObjectException, MetaException { + private MVersionTable getMSchemaVersion() throws NoSuchObjectException, MetaException { boolean committed = false; + Query query = null; List mVerTables = new ArrayList(); - try { openTransaction(); - Query query = pm.newQuery(MVersionTable.class); - + query = pm.newQuery(MVersionTable.class); try { - mVerTables = (List)query.execute(); + mVerTables = cloneQueryResult((List) query.execute()); pm.retrieveAll(mVerTables); } catch (JDODataStoreException e) { if (e.getCause() instanceof MissingTableException) { - throw new MetaException("Version table not found. " + - "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion()); + throw new MetaException("Version table not found. " + "The metastore is not upgraded to " + + MetaStoreSchemaInfo.getHiveSchemaVersion()); } else { throw e; } @@ -6735,6 +6897,9 @@ private MVersionTable getMSchemaVersion() if (!committed) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } if (mVerTables.isEmpty()) { throw new NoSuchObjectException("No matching version found"); @@ -6742,7 +6907,9 @@ private MVersionTable getMSchemaVersion() if (mVerTables.size() > 1) { String msg = "Metastore contains multiple versions (" + mVerTables.size() + ") "; for (MVersionTable version : mVerTables) { - msg += "[ version = " + version.getSchemaVersion() + ", comment = " + version.getVersionComment() + " ] "; + msg += + "[ version = " + version.getSchemaVersion() + ", comment = " + + version.getVersionComment() + " ] "; } throw new MetaException(msg.trim()); } @@ -6785,6 +6952,7 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro public boolean doesPartitionExist(String dbName, String tableName, List partVals) throws MetaException { boolean success = false; + Query query = null; try { openTransaction(); dbName = HiveStringUtils.normalizeIdentifier(dbName); @@ -6796,26 +6964,27 @@ public boolean doesPartitionExist(String dbName, String tableName, List success = commitTransaction(); return false; } - - Query query = pm.newQuery( - "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.tableName == t1 && table.database.name == t2 && partitionName == t3"); + query = + pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + + "where table.tableName == t1 && table.database.name == t2 && partitionName == t3"); query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setUnique(true); query.setResult("partitionName"); - String name = Warehouse.makePartName( - convertToFieldSchemas(mtbl.getPartitionKeys()), partVals); - String result = (String)query.execute(tableName, dbName, name); + String name = + Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), partVals); + String result = (String) query.execute(tableName, dbName, name); success = commitTransaction(); return result != null; } finally { if (!success) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } - private void debugLog(String message) { if (LOG.isDebugEnabled()) { LOG.debug(message + getCallStack()); @@ -6972,11 +7141,12 @@ public void dropFunction(String dbName, String funcName) throws MetaException, private MFunction getMFunction(String db, String function) { MFunction mfunc = null; boolean commited = false; + Query query = null; try { openTransaction(); db = HiveStringUtils.normalizeIdentifier(db); function = HiveStringUtils.normalizeIdentifier(function); - Query query = pm.newQuery(MFunction.class, "functionName == function && database.name == db"); + query = pm.newQuery(MFunction.class, "functionName == function && database.name == db"); query.declareParameters("java.lang.String function, java.lang.String db"); query.setUnique(true); mfunc = (MFunction) query.execute(function, db); @@ -6986,6 +7156,9 @@ private MFunction getMFunction(String db, String function) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return mfunc; } @@ -7007,9 +7180,9 @@ public Function getFunction(String dbName, String funcName) throws MetaException } @Override - public List getFunctions(String dbName, String pattern) - throws MetaException { + public List getFunctions(String dbName, String pattern) throws MetaException { boolean commited = false; + Query query = null; List funcs = null; try { openTransaction(); @@ -7017,25 +7190,24 @@ public Function getFunction(String dbName, String funcName) throws MetaException // Take the pattern and split it on the | to get all the composing // patterns String[] subpatterns = pattern.trim().split("\\|"); - String query = - "select functionName from org.apache.hadoop.hive.metastore.model.MFunction " - + "where database.name == dbName && ("; + String queryStr = + "select functionName from org.apache.hadoop.hive.metastore.model.MFunction " + + "where database.name == dbName && ("; boolean first = true; for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); if (!first) { - query = query + " || "; + queryStr = queryStr + " || "; } - query = query + " functionName.matches(\"" + subpattern + "\")"; + queryStr = queryStr + " functionName.matches(\"" + subpattern + "\")"; first = false; } - query = query + ")"; - - Query q = pm.newQuery(query); - q.declareParameters("java.lang.String dbName"); - q.setResult("functionName"); - q.setOrdering("functionName ascending"); - Collection names = (Collection) q.execute(dbName); + queryStr = queryStr + ")"; + query = pm.newQuery(queryStr); + query.declareParameters("java.lang.String dbName"); + query.setResult("functionName"); + query.setOrdering("functionName ascending"); + Collection names = (Collection) query.execute(dbName); funcs = new ArrayList(); for (Iterator i = names.iterator(); i.hasNext();) { funcs.add((String) i.next()); @@ -7045,6 +7217,9 @@ public Function getFunction(String dbName, String funcName) throws MetaException if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } return funcs; } @@ -7052,13 +7227,14 @@ public Function getFunction(String dbName, String funcName) throws MetaException @Override public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) { boolean commited = false; + Query query = null; try { openTransaction(); long lastEvent = rqst.getLastEvent(); - Query query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); + query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); query.declareParameters("java.lang.Long lastEvent"); query.setOrdering("eventId ascending"); - Collection events = (Collection)query.execute(lastEvent); + Collection events = (Collection) query.execute(lastEvent); commited = commitTransaction(); if (events == null) { return null; @@ -7073,6 +7249,9 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq } return result; } finally { + if (query != null) { + query.closeAll(); + } if (!commited) { rollbackTransaction(); return null; @@ -7083,9 +7262,10 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq @Override public void addNotificationEvent(NotificationEvent entry) { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MNotificationNextId.class); + query = pm.newQuery(MNotificationNextId.class); Collection ids = (Collection) query.execute(); MNotificationNextId id = null; boolean needToPersistId; @@ -7098,26 +7278,31 @@ public void addNotificationEvent(NotificationEvent entry) { } entry.setEventId(id.getNextEventId()); id.incrementEventId(); - if (needToPersistId) pm.makePersistent(id); + if (needToPersistId) + pm.makePersistent(id); pm.makePersistent(translateThriftToDb(entry)); commited = commitTransaction(); } finally { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @Override public void cleanNotificationEvents(int olderThan) { boolean commited = false; + Query query = null; try { openTransaction(); long tmp = System.currentTimeMillis() / 1000 - olderThan; - int tooOld = (tmp > Integer.MAX_VALUE) ? 0 : (int)tmp; - Query query = pm.newQuery(MNotificationLog.class, "eventTime < tooOld"); + int tooOld = (tmp > Integer.MAX_VALUE) ? 0 : (int) tmp; + query = pm.newQuery(MNotificationLog.class, "eventTime < tooOld"); query.declareParameters("java.lang.Integer tooOld"); - Collection toBeRemoved = (Collection)query.execute(tooOld); + Collection toBeRemoved = (Collection) query.execute(tooOld); if (toBeRemoved != null && toBeRemoved.size() > 0) { pm.deletePersistent(toBeRemoved); } @@ -7126,16 +7311,20 @@ public void cleanNotificationEvents(int olderThan) { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @Override public CurrentNotificationEventId getCurrentNotificationEventId() { boolean commited = false; + Query query = null; try { openTransaction(); - Query query = pm.newQuery(MNotificationNextId.class); - Collection ids = (Collection)query.execute(); + query = pm.newQuery(MNotificationNextId.class); + Collection ids = (Collection) query.execute(); long id = 0; if (ids != null && ids.size() > 0) { id = ids.iterator().next().getNextEventId() - 1; @@ -7146,6 +7335,9 @@ public CurrentNotificationEventId getCurrentNotificationEventId() { if (!commited) { rollbackTransaction(); } + if (query != null) { + query.closeAll(); + } } } @@ -7171,6 +7363,25 @@ private NotificationEvent translateDbToThrift(MNotificationLog dbEvent) { return event; } - - + /** + * Forces JDO query to iterate through the query result and copies it + * to a new list so that the JDO query object can be safely closed to + * release the resources. + * @param list Origin list + * @return Cloned list + */ + private static List cloneQueryResult(List list) { + List result = new ArrayList(); + if (list != null) { + result.addAll(list); + } + return result; + } + private static Collection cloneQueryResult(Collection list) { + Collection result = new ArrayList(); + if (list != null) { + result.addAll(list); + } + return result; + } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java new file mode 100644 index 0000000..cbdd71f --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestObjectStore { + private ObjectStore objectStore = null; + + public static class MockPartitionExpressionProxy implements PartitionExpressionProxy { + @Override + public String convertExprToFilter(byte[] expr) throws MetaException { + return null; + } + + @Override + public boolean filterPartitionsByExpr(List partColumnNames, + List partColumnTypeInfos, byte[] expr, + String defaultPartitionName, List partitionNames) + throws MetaException { + return false; + } + } + + @Before + public void setUp() { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName()); + + objectStore = new ObjectStore(); + objectStore.setConf(conf); + + Deadline.registerIfNot(100000); + } + + @After + public void tearDown() { + } + + /** + * Test database operations + */ + @Test + public void testDatabaseOps() throws MetaException, InvalidObjectException, NoSuchObjectException { + Database db1 = new Database("db1", "description", "locationurl", null); + Database db2 = new Database("db2", "description", "locationurl", null); + objectStore.createDatabase(db1); + objectStore.createDatabase(db2); + + List databases = objectStore.getAllDatabases(); + Assert.assertEquals(2, databases.size()); + Assert.assertEquals("db1", databases.get(0)); + Assert.assertEquals("db2", databases.get(1)); + + objectStore.dropDatabase("db1"); + databases = objectStore.getAllDatabases(); + Assert.assertEquals(1, databases.size()); + Assert.assertEquals("db2", databases.get(0)); + + objectStore.dropDatabase("db2"); + } + + /** + * Test table operations + */ + @Test + public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { + Database db1 = new Database("db1", "description", "locationurl", null); + objectStore.createDatabase(db1); + StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); + HashMap params = new HashMap(); + params.put("EXTERNAL", "false"); + Table tbl1 = new Table("tbl1", "db1", "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.createTable(tbl1); + + List tables = objectStore.getAllTables("db1"); + Assert.assertEquals(1, tables.size()); + Assert.assertEquals("tbl1", tables.get(0)); + + Table newTbl1 = new Table("newtbl1", "db1", "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.alterTable("db1", "tbl1", newTbl1); + tables = objectStore.getTables("db1", "new*"); + Assert.assertEquals(1, tables.size()); + Assert.assertEquals("newtbl1", tables.get(0)); + + objectStore.dropTable("db1", "newtbl1"); + tables = objectStore.getAllTables("db1"); + Assert.assertEquals(0, tables.size()); + + objectStore.dropDatabase("db1"); + } + + /** + * Tests partition operations + */ + @Test + public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { + Database db1 = new Database("db1", "description", "locationurl", null); + objectStore.createDatabase(db1); + StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); + HashMap tableParams = new HashMap(); + tableParams.put("EXTERNAL", "false"); + FieldSchema partitionKey1 = new FieldSchema("Country", "String", ""); + FieldSchema partitionKey2 = new FieldSchema("State", "String", ""); + Table tbl1 = new Table("tbl1", "db1", "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); + objectStore.createTable(tbl1); + HashMap partitionParams = new HashMap(); + partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true"); + List value1 = Arrays.asList("US", "CA"); + Partition part1 = new Partition(value1, "db1", "tbl1", 111, 111, sd, partitionParams); + objectStore.addPartition(part1); + List value2 = Arrays.asList("US", "MA"); + Partition part2 = new Partition(value2, "db1", "tbl1", 222, 222, sd, partitionParams); + objectStore.addPartition(part2); + + Deadline.startTimer("getPartition"); + List partitions = objectStore.getPartitions("db1", "tbl1", 10); + Assert.assertEquals(2, partitions.size()); + Assert.assertEquals(111, partitions.get(0).getCreateTime()); + Assert.assertEquals(222, partitions.get(1).getCreateTime()); + + objectStore.dropPartition("db1", "tbl1", value1); + partitions = objectStore.getPartitions("db1", "tbl1", 10); + Assert.assertEquals(1, partitions.size()); + Assert.assertEquals(222, partitions.get(0).getCreateTime()); + + objectStore.dropPartition("db1", "tbl1", value2); + objectStore.dropTable("db1", "tbl1"); + objectStore.dropDatabase("db1"); + } + + /** + * Test master keys operation + */ + @Test + public void testMasterKeyOps() throws MetaException, NoSuchObjectException { + int id1 = objectStore.addMasterKey("key1"); + int id2 = objectStore.addMasterKey("key2"); + + String[] keys = objectStore.getMasterKeys(); + Assert.assertEquals(2, keys.length); + Assert.assertEquals("key1", keys[0]); + Assert.assertEquals("key2", keys[1]); + + objectStore.updateMasterKey(id1, "newkey1"); + objectStore.updateMasterKey(id2, "newkey2"); + keys = objectStore.getMasterKeys(); + Assert.assertEquals(2, keys.length); + Assert.assertEquals("newkey1", keys[0]); + Assert.assertEquals("newkey2", keys[1]); + + objectStore.removeMasterKey(id1); + keys = objectStore.getMasterKeys(); + Assert.assertEquals(1, keys.length); + Assert.assertEquals("newkey2", keys[0]); + + objectStore.removeMasterKey(id2); + } +}