Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (revision 1134205) +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (working copy) @@ -251,6 +251,7 @@ String partName = "ds=2008-07-01 14%3A13%3A12/hr=14"; String part2Name = "ds=2008-07-01 14%3A13%3A12/hr=15"; String part3Name ="ds=2008-07-02 14%3A13%3A12/hr=15"; + String part4Name ="ds=2008-07-03 14%3A13%3A12/hr=151"; part_get = client.getPartition(dbName, tblName, partName); assertTrue("Partitions are not the same", part.equals(part_get)); @@ -275,6 +276,14 @@ assertTrue("Should have returned 2 partition names", partialNames.size() == 2); assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + partNames.add(part3Name); + partNames.add(part4Name); + partialVals.clear(); + partialVals.add(""); + partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1); + assertTrue("Should have returned 4 partition names", partialNames.size() == 4); + assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + // Test partition listing with a partial spec - hr is specified but ds is not parts.clear(); parts.add(part2); Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 1134205) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy) @@ -400,6 +400,27 @@ return FileUtils.makePartName(colNames, vals); } + /** + * Makes a valid partition name. + * @param partCols The partition columns + * @param vals The partition values + * @param defaultStr + * The default name given to a partition value if the respective value is empty or null. + * @return An escaped, valid partition name. + * @throws MetaException + */ + public static String makePartName(List partCols, + List vals, String defaultStr) throws MetaException { + if ((partCols.size() != vals.size()) || (partCols.size() == 0)) { + throw new MetaException("Invalid partition key & values"); + } + List colNames = new ArrayList(); + for (FieldSchema col: partCols) { + colNames.add(col.getName()); + } + return FileUtils.makePartName(colNames, vals, defaultStr); + } + public static List getPartValuesFromPartName(String partName) throws MetaException { LinkedHashMap partSpec = Warehouse.makeSpecFromName(partName); Index: metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (revision 1134205) +++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (working copy) @@ -232,6 +232,22 @@ public abstract List getPartitionsWithAuth(String dbName, String tblName, short maxParts, String userName, List groupNames) - throws MetaException, NoSuchObjectException, InvalidObjectException;; + throws MetaException, NoSuchObjectException, InvalidObjectException; + /** + * Returns a list of partitions that match a given partial specification + * @param db_name + * The name of the database which has the partitions + * @param tbl_name + * The name of the table which has the partitions + * @param part_vals + * A list of values for partitions in order of the partition keys. + * Entries can be empty if you only want to specify latter partitions. + * @param max_parts + * The maximum number of partitions to return + */ + public abstract List getPartitionNamesPs(String db_name, String tbl_name, + List part_vals, short max_parts) + throws MetaException, NoSuchObjectException ; + } Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1134205) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -2172,34 +2172,23 @@ final String tbl_name, final List part_vals, final short max_parts) throws MetaException, TException { startPartitionFunction("get_partitions_names_ps", db_name, tbl_name, part_vals); + List ret; try { - Table t; - try { - t = get_table(db_name, tbl_name); - } catch (NoSuchObjectException e) { - throw new MetaException(e.getMessage()); - } - - List partNames = get_partition_names(db_name, tbl_name, max_parts); - List filteredPartNames = new ArrayList(); - - for(String name : partNames) { - LinkedHashMap spec = Warehouse.makeSpecFromName(name); - List vals = new ArrayList(); - // Since we are iterating through a LinkedHashMap, iteration should - // return the partition values in the correct order for comparison. - for (String val : spec.values()) { - vals.add(val); + ret = executeWithRetry(new Command>() { + @Override + public List run(RawStore ms) throws Exception { + return ms.getPartitionNamesPs(db_name, tbl_name, part_vals, max_parts); } - if (MetaStoreUtils.pvalMatches(part_vals, vals)) { - filteredPartNames.add(name); - } - } - - return filteredPartNames; + }); + } catch (MetaException e) { + throw e; + } catch (Exception e) { + assert(e instanceof RuntimeException); + throw (RuntimeException)e; } finally { endFunction("get_partitions_names_ps"); } + return ret; } @Override Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1134205) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -1275,6 +1275,57 @@ return pns; } + @Override + public List getPartitionNamesPs(String dbName, String tableName, + List part_vals, short max_parts) throws MetaException, NoSuchObjectException { + List partitionNames = new ArrayList(); + boolean success = false; + try { + openTransaction(); + LOG.debug("Executing getPartitionNamesPs"); + dbName = dbName.toLowerCase().trim(); + tableName = tableName.toLowerCase().trim(); + Table table = getTable(dbName, tableName); + + List partCols = table.getPartitionKeys(); + int numPartKeys = partCols.size(); + if (part_vals.size() > numPartKeys) { + throw new MetaException("Incorrect number of partition values"); + } + + partCols = partCols.subList(0, part_vals.size()); + String partNameMatcher = Warehouse.makePartName(partCols, part_vals, ".*"); + if (part_vals.size() < numPartKeys) { + partNameMatcher += ".*"; + } + + Query q = pm.newQuery(MPartition.class); + StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); + queryFilter.append(" && table.tableName == tableName"); + queryFilter.append(" && partitionName.matches(partialRegex)"); + q.setFilter(queryFilter.toString()); + q.setResult("partitionName"); + q.declareParameters("java.lang.String dbName, java.lang.String tableName, java.lang.String partialRegex"); + + if( max_parts >= 0 ) { + //User specified a row limit, set it on the Query + q.setRange(0, max_parts); + } + + Collection names = (Collection) q.execute(dbName, tableName, partNameMatcher); + for (Object o : names) { + partitionNames.add((String) o); + } + + success = commitTransaction(); + } finally { + if (!success) { + rollbackTransaction(); + } + } + return partitionNames; + } + // TODO:pc implement max private List listMPartitions(String dbName, String tableName, int max) { @@ -1477,7 +1528,6 @@ Map params = new HashMap(); String queryFilterString = makeQueryFilterString(mtable, filter, params); - Query query = pm.newQuery( "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + "where " + queryFilterString); Index: common/src/java/org/apache/hadoop/hive/common/FileUtils.java =================================================================== --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java (revision 1134205) +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java (working copy) @@ -107,6 +107,28 @@ return name.toString(); } + /** + * Makes a valid partition name. + * @param partCols The partition keys' names + * @param vals The partition values + * @param defaultStr + * The default name given to a partition value if the respective value is empty or null. + * @return An escaped, valid partition name. + */ + public static String makePartName(List partCols, List vals, + String defaultStr) { + StringBuilder name = new StringBuilder(); + for (int i = 0; i < partCols.size(); i++) { + if (i > 0) { + name.append(Path.SEPARATOR); + } + name.append(escapePathName((partCols.get(i)).toLowerCase(), defaultStr)); + name.append('='); + name.append(escapePathName(vals.get(i), defaultStr)); + } + return name.toString(); + } + // NOTE: This is for generating the internal path name for partitions. Users // should always use the MetaStore API to get the path name for a partition. // Users should not directly take partition values and turn it into a path @@ -121,7 +143,7 @@ for (char c = 0; c < ' '; c++) { charToEscape.set(c); } - + /** * ASCII 01-1F are HTTP control characters that need to be escaped. * \u000A and \u000D are \n and \r, respectively. @@ -163,6 +185,31 @@ return sb.toString(); } + /** + * Escapes a path name. + * @param path The path to escape. + * @param defaultPath + * The default name for the path, if the given path is empty or null. + * @return An escaped path name. + */ + public static String escapePathName(String path, String defaultPath) { + if (path == null || path.length() == 0) { + return defaultPath; + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < path.length(); i++) { + char c = path.charAt(i); + if (needsEscaping(c)) { + sb.append('%'); + sb.append(String.format("%1$02X", (int) c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } + public static String unescapePathName(String path) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < path.length(); i++) {