Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (revision 1135227) +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (working copy) @@ -251,6 +251,7 @@ String partName = "ds=2008-07-01 14%3A13%3A12/hr=14"; String part2Name = "ds=2008-07-01 14%3A13%3A12/hr=15"; String part3Name ="ds=2008-07-02 14%3A13%3A12/hr=15"; + String part4Name ="ds=2008-07-03 14%3A13%3A12/hr=151"; part_get = client.getPartition(dbName, tblName, partName); assertTrue("Partitions are not the same", part.equals(part_get)); @@ -275,6 +276,14 @@ assertTrue("Should have returned 2 partition names", partialNames.size() == 2); assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + partNames.add(part3Name); + partNames.add(part4Name); + partialVals.clear(); + partialVals.add(""); + partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1); + assertTrue("Should have returned 4 partition names", partialNames.size() == 4); + assertTrue("Not all part names returned", partialNames.containsAll(partNames)); + // Test partition listing with a partial spec - hr is specified but ds is not parts.clear(); parts.add(part2); Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 1135227) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy) @@ -390,6 +390,20 @@ public static String makePartName(List partCols, List vals) throws MetaException { + return makePartName(partCols, vals, null); + } + + /** + * Makes a valid partition name. + * @param partCols The partition columns + * @param vals The partition values + * @param defaultStr + * The default name given to a partition value if the respective value is empty or null. + * @return An escaped, valid partition name. + * @throws MetaException + */ + public static String makePartName(List partCols, + List vals, String defaultStr) throws MetaException { if ((partCols.size() != vals.size()) || (partCols.size() == 0)) { throw new MetaException("Invalid partition key & values"); } @@ -397,7 +411,7 @@ for (FieldSchema col: partCols) { colNames.add(col.getName()); } - return FileUtils.makePartName(colNames, vals); + return FileUtils.makePartName(colNames, vals, defaultStr); } public static List getPartValuesFromPartName(String partName) Index: metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (revision 1135227) +++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (working copy) @@ -232,6 +232,49 @@ public abstract List getPartitionsWithAuth(String dbName, String tblName, short maxParts, String userName, List groupNames) - throws MetaException, NoSuchObjectException, InvalidObjectException;; + throws MetaException, NoSuchObjectException, InvalidObjectException; -} + /** + * Lists partition names that match a given partial specification + * @param db_name + * The name of the database which has the partitions + * @param tbl_name + * The name of the table which has the partitions + * @param part_vals + * A partial list of values for partitions in order of the table's partition keys. + * Entries can be empty if you only want to specify latter partitions. + * @param max_parts + * The maximum number of partitions to return + * @return A list of partition names that match the partial spec. + * @throws MetaException + * @throws NoSuchObjectException + */ + public abstract List listPartitionNamesPs(String db_name, String tbl_name, + List part_vals, short max_parts) + throws MetaException; + + /** + * Lists partitions that match a given partial specification and sets their auth privileges. + * If userName and groupNames null, then no auth privileges are set. + * @param db_name + * The name of the database which has the partitions + * @param tbl_name + * The name of the table which has the partitions + * @param part_vals + * A partial list of values for partitions in order of the table's partition keys + * Entries can be empty if you need to specify latter partitions. + * @param max_parts + * The maximum number of partitions to return + * @param userName + * The user name for the partition for authentication privileges + * @param groupNames + * The groupNames for the partition for authentication privileges + * @return A list of partitions that match the partial spec. + * @throws MetaException + * @throws NoSuchObjectException + * @throws InvalidObjectException + */ + public abstract List listPartitionsPsWithAuth(String db_name, String tbl_name, + List part_vals, short max_parts, String userName, List groupNames) + throws MetaException, InvalidObjectException; +} \ No newline at end of file Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1135227) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -2124,7 +2124,7 @@ final short max_parts) throws MetaException, TException { startPartitionFunction("get_partitions_ps", db_name, tbl_name, part_vals); try { - return this.get_partitions_ps_with_auth(db_name, tbl_name, part_vals, + return get_partitions_ps_with_auth(db_name, tbl_name, part_vals, max_parts, null, null); } finally { @@ -2139,32 +2139,26 @@ final List groupNames) throws MetaException, TException { startPartitionFunction("get_partitions_ps_with_auth", db_name, tbl_name, part_vals); - List parts = null; - List matchingParts = new ArrayList(); - + List ret; try { - // This gets all the partitions and then filters based on the specified - // criteria. An alternative approach would be to get all the partition - // names, do the filtering on the names, and get the partition for each - // of the names. that match. - - try { - parts = get_partitions(db_name, tbl_name, (short) -1); - } catch (NoSuchObjectException e) { - throw new MetaException(e.getMessage()); - } - - for (Partition p : parts) { - if (MetaStoreUtils.pvalMatches(part_vals, p.getValues())) { - matchingParts.add(p); + ret = executeWithRetry(new Command>() { + @Override + public List run(RawStore ms) throws Exception { + return ms.listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, + userName, groupNames); } - } - - return matchingParts; - } - finally { + }); + } catch (MetaException e) { + throw e; + } catch (InvalidObjectException e) { + throw new MetaException(e.getMessage()); + } catch (Exception e) { + assert(e instanceof RuntimeException); + throw (RuntimeException)e; + } finally { endFunction("get_partitions_ps_with_auth"); } + return ret; } @Override @@ -2172,34 +2166,23 @@ final String tbl_name, final List part_vals, final short max_parts) throws MetaException, TException { startPartitionFunction("get_partitions_names_ps", db_name, tbl_name, part_vals); + List ret; try { - Table t; - try { - t = get_table(db_name, tbl_name); - } catch (NoSuchObjectException e) { - throw new MetaException(e.getMessage()); - } - - List partNames = get_partition_names(db_name, tbl_name, max_parts); - List filteredPartNames = new ArrayList(); - - for(String name : partNames) { - LinkedHashMap spec = Warehouse.makeSpecFromName(name); - List vals = new ArrayList(); - // Since we are iterating through a LinkedHashMap, iteration should - // return the partition values in the correct order for comparison. - for (String val : spec.values()) { - vals.add(val); + ret = executeWithRetry(new Command>() { + @Override + public List run(RawStore ms) throws Exception { + return ms.listPartitionNamesPs(db_name, tbl_name, part_vals, max_parts); } - if (MetaStoreUtils.pvalMatches(part_vals, vals)) { - filteredPartNames.add(name); - } - } - - return filteredPartNames; + }); + } catch (MetaException e) { + throw e; + } catch (Exception e) { + assert(e instanceof RuntimeException); + throw (RuntimeException)e; } finally { endFunction("get_partitions_names_ps"); } + return ret; } @Override Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1135227) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -1275,6 +1275,120 @@ return pns; } + /** + * Retrieves a Collection of partition-related results from the database that match + * the partial specification given for a specific table. + * @param dbName the name of the database + * @param tableName the name of the table + * @param part_vals the partial specification values + * @param max_parts the maximum number of partitions to return + * @param resultsCol the metadata column of the data to return, e.g. partitionName, etc. + * if resultsCol is empty or null, a collection of MPartition objects is returned + * @results A Collection of partition-related items from the db that match the partial spec + * for a table. The type of each item in the collection corresponds to the column + * you want results for. E.g., if resultsCol is partitionName, the Collection + * has types of String, and if resultsCol is null, the types are MPartition. + */ + private Collection getPartitionPsQueryResults(String dbName, String tableName, + List part_vals, short max_parts, String resultsCol) + throws MetaException { + dbName = dbName.toLowerCase().trim(); + tableName = tableName.toLowerCase().trim(); + Table table = getTable(dbName, tableName); + + List partCols = table.getPartitionKeys(); + int numPartKeys = partCols.size(); + if (part_vals.size() > numPartKeys) { + throw new MetaException("Incorrect number of partition values"); + } + + partCols = partCols.subList(0, part_vals.size()); + //Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // where partVal is either the escaped partition value given as input, + // or a regex of the form ".*" + //This works because the "=" and "/" separating key names and partition key/values + // are not escaped. + String partNameMatcher = Warehouse.makePartName(partCols, part_vals, ".*"); + //add ".*" to the regex to match anything else afterwards the partial spec. + if (part_vals.size() < numPartKeys) { + partNameMatcher += ".*"; + } + + Query q = pm.newQuery(MPartition.class); + StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); + queryFilter.append(" && table.tableName == tableName"); + queryFilter.append(" && partitionName.matches(partialRegex)"); + q.setFilter(queryFilter.toString()); + q.declareParameters("java.lang.String dbName, " + + "java.lang.String tableName, java.lang.String partialRegex"); + + if( max_parts >= 0 ) { + //User specified a row limit, set it on the Query + q.setRange(0, max_parts); + } + if (resultsCol != null && !resultsCol.isEmpty()) { + q.setResult(resultsCol); + } + + return (Collection) q.execute(dbName, tableName, partNameMatcher); + } + + @Override + public List listPartitionsPsWithAuth(String db_name, String tbl_name, + List part_vals, short max_parts, String userName, List groupNames) + throws MetaException, InvalidObjectException { + List partitions = new ArrayList(); + boolean success = false; + try { + openTransaction(); + LOG.debug("executing listPartitionNamesPsWithAuth"); + Collection parts = getPartitionPsQueryResults(db_name, tbl_name, + part_vals, max_parts, null); + MTable mtbl = getMTable(db_name, tbl_name); + for (Object o : parts) { + Partition part = convertToPart((MPartition) o); + //set auth privileges + if (null != userName && null != groupNames && + "TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { + String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl + .getPartitionKeys()), part.getValues()); + PrincipalPrivilegeSet partAuth = getPartitionPrivilegeSet(db_name, + tbl_name, partName, userName, groupNames); + part.setPrivileges(partAuth); + } + partitions.add(part); + } + success = commitTransaction(); + } finally { + if (!success) { + rollbackTransaction(); + } + } + return partitions; + } + + @Override + public List listPartitionNamesPs(String dbName, String tableName, + List part_vals, short max_parts) throws MetaException { + List partitionNames = new ArrayList(); + boolean success = false; + try { + openTransaction(); + LOG.debug("Executing listPartitionNamesPs"); + Collection names = getPartitionPsQueryResults(dbName, tableName, + part_vals, max_parts, "partitionName"); + for (Object o : names) { + partitionNames.add((String) o); + } + success = commitTransaction(); + } finally { + if (!success) { + rollbackTransaction(); + } + } + return partitionNames; + } + // TODO:pc implement max private List listMPartitions(String dbName, String tableName, int max) { @@ -1477,7 +1591,6 @@ Map params = new HashMap(); String queryFilterString = makeQueryFilterString(mtable, filter, params); - Query query = pm.newQuery( "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + "where " + queryFilterString); Index: common/src/java/org/apache/hadoop/hive/common/FileUtils.java =================================================================== --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java (revision 1135227) +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java (working copy) @@ -94,15 +94,27 @@ public static String makePartName(List partCols, List vals) { + return makePartName(partCols, vals, null); + } + /** + * Makes a valid partition name. + * @param partCols The partition keys' names + * @param vals The partition values + * @param defaultStr + * The default name given to a partition value if the respective value is empty or null. + * @return An escaped, valid partition name. + */ + public static String makePartName(List partCols, List vals, + String defaultStr) { StringBuilder name = new StringBuilder(); for (int i = 0; i < partCols.size(); i++) { if (i > 0) { name.append(Path.SEPARATOR); } - name.append(escapePathName((partCols.get(i)).toLowerCase())); + name.append(escapePathName((partCols.get(i)).toLowerCase(), defaultStr)); name.append('='); - name.append(escapePathName(vals.get(i))); + name.append(escapePathName(vals.get(i), defaultStr)); } return name.toString(); } @@ -121,7 +133,7 @@ for (char c = 0; c < ' '; c++) { charToEscape.set(c); } - + /** * ASCII 01-1F are HTTP control characters that need to be escaped. * \u000A and \u000D are \n and \r, respectively. @@ -143,11 +155,28 @@ } public static String escapePathName(String path) { + return escapePathName(path, null); + } - // __HIVE_DEFAULT_NULL__ is the system default value for null and empty string. We should + /** + * Escapes a path name. + * @param path The path to escape. + * @param defaultPath + * The default name for the path, if the given path is empty or null. + * @return An escaped path name. + */ + public static String escapePathName(String path, String defaultPath) { + + // __HIVE_DEFAULT_NULL__ is the system default value for null and empty string. // TODO: we should allow user to specify default partition or HDFS file location. if (path == null || path.length() == 0) { - return "__HIVE_DEFAULT_PARTITION__"; + if (defaultPath == null) { + //previously, when path is empty or null and no default path is specified, + // __HIVE_DEFAULT_PARTITION__ was the return value for escapePathName + return "__HIVE_DEFAULT_PARTITION__"; + } else { + return defaultPath; + } } StringBuilder sb = new StringBuilder();