Index: webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java =================================================================== --- webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (revision 1425093) +++ webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (working copy) @@ -215,7 +215,7 @@ HCatPartition ptn = client.getPartition(dbName, tableName, firstPtn); assertTrue(ptn != null); - client.dropPartition(dbName, tableName, firstPtn, true); + client.dropPartitions(dbName, tableName, firstPtn, true); ptnList = client.listPartitionsByFilter(dbName, tableName, null); assertTrue(ptnList.size() == 2); @@ -503,6 +503,104 @@ } @Test + public void testGetPartitionsWithPartialSpec() throws Exception { + try { + HCatClient client = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "myDb"; + final String tableName = "myTable"; + + client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + client.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, "")); + + List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), + new HCatFieldSchema("grid", Type.STRING, "")); + + client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList(partitionSchema)).build()); + + Map partitionSpec = new HashMap(); + partitionSpec.put("grid", "AB"); + partitionSpec.put("dt", "2011_12_31"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("grid", "AB"); + partitionSpec.put("dt", "2012_01_01"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("dt", "2012_01_01"); + partitionSpec.put("grid", "OB"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("dt", "2012_01_01"); + partitionSpec.put("grid", "XB"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + + Map partialPartitionSpec = new HashMap(); + partialPartitionSpec.put("dt", "2012_01_01"); + + List partitions = client.getPartitions(dbName, tableName, partialPartitionSpec); + assertEquals("Unexpected number of partitions.", 3, partitions.size()); + assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "AB"}, partitions.get(0).getValues().toArray()); + assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "OB"}, partitions.get(1).getValues().toArray()); + assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "XB"}, partitions.get(2).getValues().toArray()); + + client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE); + } + catch (Exception unexpected) { + LOG.error("Unexpected exception!", unexpected); + assertTrue("Unexpected exception! " + unexpected.getMessage(), false); + } + } + + @Test + public void testDropPartitionsWithPartialSpec() throws Exception { + try { + HCatClient client = HCatClient.create(new Configuration(hcatConf)); + final String dbName = "myDb"; + final String tableName = "myTable"; + + client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); + + client.createDatabase(HCatCreateDBDesc.create(dbName).build()); + List columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), + new HCatFieldSchema("bar", Type.STRING, "")); + + List partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""), + new HCatFieldSchema("grid", Type.STRING, "")); + + client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList(partitionSchema)).build()); + + Map partitionSpec = new HashMap(); + partitionSpec.put("grid", "AB"); + partitionSpec.put("dt", "2011_12_31"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("grid", "AB"); + partitionSpec.put("dt", "2012_01_01"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("dt", "2012_01_01"); + partitionSpec.put("grid", "OB"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + partitionSpec.put("dt", "2012_01_01"); + partitionSpec.put("grid", "XB"); + client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build()); + + Map partialPartitionSpec = new HashMap(); + partialPartitionSpec.put("dt", "2012_01_01"); + + client.dropPartitions(dbName, tableName, partialPartitionSpec, true); + + List partitions = client.getPartitions(dbName, tableName); + assertEquals("Unexpected number of partitions.", 1, partitions.size()); + assertArrayEquals("Mismatched partition.", new String[]{"2011_12_31", "AB"}, partitions.get(0).getValues().toArray()); + + client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE); + } + catch (Exception unexpected) { + LOG.error("Unexpected exception!", unexpected); + assertTrue("Unexpected exception! " + unexpected.getMessage(), false); + } + } + + @Test public void testPartitionSchema() throws Exception { try { HCatClient client = HCatClient.create(new Configuration(hcatConf)); Index: webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java =================================================================== --- webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (revision 1425093) +++ webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (working copy) @@ -327,12 +327,44 @@ } @Override + public List getPartitions(String dbName, String tblName, Map partitionSpec) throws HCatException { + return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec)); + } + + private static String getFilterString(Map partitionSpec) { + final String AND = " AND "; + StringBuilder filter = new StringBuilder(); + for (Map.Entry entry : partitionSpec.entrySet()) { + filter.append(entry.getKey()).append("=").append("\"").append(entry.getValue()).append("\"").append(AND); + } + + int length = filter.toString().length(); + if (length > 0) + filter.delete(length - AND.length(), length); + + return filter.toString(); + } + + @Override public HCatPartition getPartition(String dbName, String tableName, Map partitionSpec) throws HCatException { HCatPartition partition = null; try { + List partitionColumns = getTable(checkDB(dbName), tableName).getPartCols(); + if (partitionColumns.size() != partitionSpec.size()) { + throw new HCatException("Partition-spec doesn't have the right number of partition keys."); + } + ArrayList ptnValues = new ArrayList(); - ptnValues.addAll(partitionSpec.values()); + for (HCatFieldSchema partitionColumn : partitionColumns) { + String partKey = partitionColumn.getName(); + if (partitionSpec.containsKey(partKey)) { + ptnValues.add(partitionSpec.get(partKey)); // Partition-keys added in order. + } + else { + throw new HCatException("Invalid partition-key specified: " + partKey); + } + } Partition hivePartition = hmsClient.getPartition(checkDB(dbName), tableName, ptnValues); if (hivePartition != null) { @@ -383,19 +415,22 @@ } @Override - public void dropPartition(String dbName, String tableName, + public void dropPartitions(String dbName, String tableName, Map partitionSpec, boolean ifExists) throws HCatException { try { - List ptnValues = new ArrayList(); - ptnValues.addAll(partitionSpec.values()); - hmsClient.dropPartition(checkDB(dbName), tableName, ptnValues, - ifExists); + dbName = checkDB(dbName); + List partitions = hmsClient.listPartitionsByFilter(dbName, tableName, + getFilterString(partitionSpec), (short)-1); + + for (Partition partition : partitions) { + dropPartition(partition, ifExists); + } + } catch (NoSuchObjectException e) { - if (!ifExists) { - throw new ObjectNotFoundException( - "NoSuchObjectException while dropping partition.", e); - } + throw new ObjectNotFoundException( + "NoSuchObjectException while dropping partition. " + + "Either db(" + dbName + ") or table(" + tableName + ") missing.", e); } catch (MetaException e) { throw new HCatException("MetaException while dropping partition.", e); @@ -405,6 +440,18 @@ } } + private void dropPartition(Partition partition, boolean ifExists) + throws HCatException, MetaException, TException { + try { + hmsClient.dropPartition(partition.getDbName(), partition.getTableName(), partition.getValues()); + } catch (NoSuchObjectException e) { + if (!ifExists) { + throw new ObjectNotFoundException( + "NoSuchObjectException while dropping partition: " + partition.getValues(), e); + } + } + } + @Override public List listPartitionsByFilter(String dbName, String tblName, String filter) throws HCatException { Index: webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java =================================================================== --- webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (revision 1425093) +++ webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (working copy) @@ -204,11 +204,30 @@ throws HCatException; /** + * Gets all the partitions that match the specified (and possibly partial) partition specification. + * A partial partition-specification is one where not all partition-keys have associated values. For example, + * for a table ('myDb.myTable') with 2 partition keys (dt string, region string), + * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then, + * 1. Complete partition spec: getPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would return 1 partition. + * 2. Partial partition spec: getPartitions('myDb', 'myTable', {dt='20120101'}) would return all 3 partitions, + * with dt='20120101' (i.e. region = 'us', 'uk' and 'in'). + * @param dbName The name of the database. + * @param tableName The name of the table. + * @param partitionSpec The partition specification. (Need not include all partition keys.) + * @return A list of partitions. + * @throws HCatException + */ + public abstract List getPartitions(String dbName, String tableName, + Map partitionSpec) throws HCatException; + + + /** * Gets the partition. * * @param dbName The database name. * @param tableName The table name. - * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. + * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. All partition-key-values + * must be specified. * @return An instance of HCatPartitionInfo. * @throws HCatException */ @@ -235,15 +254,20 @@ throws HCatException; /** - * Drops partition. - * + * Drops partition(s) that match the specified (and possibly partial) partition specification. + * A partial partition-specification is one where not all partition-keys have associated values. For example, + * for a table ('myDb.myTable') with 2 partition keys (dt string, region string), + * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then, + * 1. Complete partition spec: dropPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would drop 1 partition. + * 2. Partial partition spec: dropPartitions('myDb', 'myTable', {dt='20120101'}) would drop all 3 partitions, + * with dt='20120101' (i.e. region = 'us', 'uk' and 'in'). * @param dbName The database name. * @param tableName The table name. * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. * @param ifExists Hive returns an error if the partition specified does not exist, unless ifExists is set to true. * @throws HCatException */ - public abstract void dropPartition(String dbName, String tableName, + public abstract void dropPartitions(String dbName, String tableName, Map partitionSpec, boolean ifExists) throws HCatException;