commit 8caab8e40550916051e792ae1065ccf1440c869c Author: Alan Gates Date: Tue Feb 10 13:33:26 2015 -0800 HIVE-9641 WIP HIVE-9641 WIP, still need to add tests for scanPartitions HIVE-9641 Got tests passing. diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java index f33da21..e7e0178 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java @@ -354,6 +354,84 @@ public void createPartition() throws Exception { Assert.assertEquals("fred", p.getValues().get(0)); } + @Test + public void addPartitions() throws Exception { + String dbName = "default"; + String tableName = "addParts"; + int startTime = (int)(System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "nocomment")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, + serde, null, null, emptyParameters); + List partCols = new ArrayList(); + partCols.add(new FieldSchema("pc", "string", "")); + Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + store.createTable(table); + + List partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan"); + List partitions = new ArrayList(); + for (String val : partVals) { + List vals = new ArrayList(); + vals.add(val); + StorageDescriptor psd = new StorageDescriptor(sd); + psd.setLocation("file:/tmp/pc=" + val); + Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, + emptyParameters); + partitions.add(part); + } + store.addPartitions(dbName, tableName, partitions); + + List partNames = store.listPartitionNames(dbName, tableName, (short) -1); + Assert.assertEquals(5, partNames.size()); + String[] names = partNames.toArray(new String[partNames.size()]); + Arrays.sort(names); + String[] canonicalNames = partVals.toArray(new String[partVals.size()]); + for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i]; + Assert.assertArrayEquals(canonicalNames, names); + } + + @Test + public void alterPartitions() throws Exception { + String dbName = "default"; + String tableName = "alterParts"; + int startTime = (int)(System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "nocomment")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, + serde, null, null, emptyParameters); + List partCols = new ArrayList(); + partCols.add(new FieldSchema("pc", "string", "")); + Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + store.createTable(table); + + List partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan"); + List partitions = new ArrayList(); + List> allVals = new ArrayList>(); + for (String val : partVals) { + List vals = new ArrayList(); + allVals.add(vals); + vals.add(val); + StorageDescriptor psd = new StorageDescriptor(sd); + psd.setLocation("file:/tmp/pc=" + val); + Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, + emptyParameters); + partitions.add(part); + } + store.addPartitions(dbName, tableName, partitions); + + for (Partition p : partitions) p.setLastAccessTime(startTime + 10); + store.alterPartitions(dbName, tableName, allVals, partitions); + + partitions = store.getPartitions(dbName, tableName, -1); + for (Partition part : partitions) { + Assert.assertEquals(startTime + 10, part.getLastAccessTime()); + } + } + // TODO - Fix this and the next test. They depend on test execution order and are bogus. @Test public void createManyPartitions() throws Exception { @@ -489,6 +567,70 @@ public void listPartitions() throws Exception { } @Test + public void listPartitionsWithPs() throws Exception { + String dbName = "default"; + String tableName = "listPartsPs"; + int startTime = (int)(System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "nocomment")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, + serde, null, null, emptyParameters); + List partCols = new ArrayList(); + partCols.add(new FieldSchema("ds", "string", "")); + partCols.add(new FieldSchema("region", "string", "")); + Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + store.createTable(table); + + String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"}, + {"tomorrow", "north america"}, {"tomorrow", "europe"}}; + for (String[] pv : partVals) { + List vals = new ArrayList(); + for (String v : pv) vals.add(v); + StorageDescriptor psd = new StorageDescriptor(sd); + psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]); + Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, + emptyParameters); + store.addPartition(part); + } + + // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway. + // Test the case where we completely specify the partition + List partitionNames = + store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1); + Assert.assertEquals(1, partitionNames.size()); + Assert.assertEquals("ds=today/region=north america", partitionNames.get(0)); + + // Leave off the last value of the partition + partitionNames = + store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1); + Assert.assertEquals(2, partitionNames.size()); + String[] names = partitionNames.toArray(new String[partitionNames.size()]); + Arrays.sort(names); + Assert.assertArrayEquals(new String[] {"ds=today/region=europe", + "ds=today/region=north america"}, names); + + // Put a star in the last value of the partition + partitionNames = + store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1); + Assert.assertEquals(2, partitionNames.size()); + names = partitionNames.toArray(new String[partitionNames.size()]); + Arrays.sort(names); + Assert.assertArrayEquals(new String[] {"ds=today/region=europe", + "ds=today/region=north america"}, names); + + // Put a star in the first value of the partition + partitionNames = + store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1); + Assert.assertEquals(2, partitionNames.size()); + names = partitionNames.toArray(new String[partitionNames.size()]); + Arrays.sort(names); + Assert.assertArrayEquals(new String[] {"ds=today/region=europe", + "ds=tomorrow/region=europe"}, names); + } + + @Test public void dropPartition() throws Exception { String dbName = "default"; String tableName = "myparttable2"; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java index d802165..e0ccbd5 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore.hbase; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,6 +47,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -80,7 +82,7 @@ @VisibleForTesting final static String NO_CACHE_CONF = "no.use.cache"; private final static byte[] CATALOG_COL = "cat".getBytes(HBaseUtils.ENCODING); private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING); - private final static int tablesToCache = 10; + private final static int TABLES_TO_CACHE = 10; // TODO Add privileges as a second column in the CATALOG_CF @@ -190,7 +192,7 @@ private HBaseReadWrite(Configuration configuration) { partCache = new BogusPartitionCache(); statsCache = StatsCache.getBogusStatsCache(); } else { - tableCache = new ObjectCache, Table>(tablesToCache, tableHits, + tableCache = new ObjectCache, Table>(TABLES_TO_CACHE, tableHits, tableMisses, tableOverflows); sdCache = new ObjectCache(sdsCacheSize, sdHits, sdMisses, sdOverflows); @@ -334,6 +336,26 @@ void putPartition(Partition partition) throws IOException { } /** + * Add a group of partitions + * @param partitions list of partitions to add + * @throws IOException + */ + void putPartitions(List partitions) throws IOException { + List puts = new ArrayList(partitions.size()); + for (Partition partition : partitions) { + PartitionWritable part = new PartitionWritable(partition); + byte[] key = buildPartitionKey(part); + byte[] serialized = HBaseUtils.serialize(part); + Put p = new Put(key); + p.add(CATALOG_CF, CATALOG_COL, serialized); + puts.add(p); + partCache.put(partition.getDbName(), partition.getTableName(), partition); + } + getHTable(PART_TABLE).put(puts); + flush(); + } + + /** * Find all the partitions in a table. * @param dbName name of the database the table is in * @param tableName table name @@ -360,45 +382,82 @@ void putPartition(Partition partition) throws IOException { * Scan partitions based on partial key information. * @param dbName name of database, required * @param tableName name of table, required - * @param partVals partial specification of values. Any values that are unknown can be left - * null in the list. For example, if a table had two partition columns date + * @param partVals partial specification of values. Any values that are unknown can instead be + * a '*'. For example, if a table had two partition columns date * and region (in that order), and partitions ('today', 'na'), ('today', 'eu'), - * ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today'] would return - * ('today', 'na') and ('today', 'eu') while passing [null, 'eu'] would return - * ('today', 'eu') and ('tomorrow', 'eu') + * ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today', '*'] would return + * ('today', 'na') and ('today', 'eu') while passing ['*', 'eu'] would return + * ('today', 'eu') and ('tomorrow', 'eu'). Also the list can terminate early, + * which will be the equivalent of adding '*' for all non-included values. + * I.e. ['today'] is the same as ['today', '*']. * @param maxPartitions Maximum number of entries to return. * @return list of partitions that match the specified information * @throws IOException + * @throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException if the table containing + * the partitions can't be found. */ List scanPartitions(String dbName, String tableName, List partVals, - int maxPartitions) throws IOException { + int maxPartitions) throws IOException, NoSuchObjectException { + // First, build as much of the key as we can so that we make the scan as tight as possible. + List keyElements = new ArrayList(); + keyElements.add(dbName); + keyElements.add(tableName); + + int firstStar = -1; + for (int i = 0; i < partVals.size(); i++) { + if ("*".equals(partVals.get(i))) { + firstStar = i; + break; + } else { + keyElements.add(partVals.get(i)); + } + } + byte[] keyPrefix; - if (partVals == null || partVals.size() == 0) { - keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName); - return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); + // We need to fetch the table to determine if the user fully specified the partitions or + // not, as it affects how we build the key. + Table table = getTable(dbName, tableName); + if (table == null) { + throw new NoSuchObjectException("Unable to find table " + dbName + "." + tableName); } - int firstNull = 0; - for (; firstNull < partVals.size(); firstNull++) { - if (partVals.get(firstNull) == null) break; + if (partVals.size() == table.getPartitionKeys().size()) { + keyPrefix = HBaseUtils.buildKey(keyElements.toArray(new String[keyElements.size()])); + } else { + keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(keyElements.toArray( + new String[keyElements.size()])); } - if (firstNull == partVals.size()) { - keyPrefix = buildPartitionKey(dbName, tableName, partVals); - return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); + + // Now, build a filter out of the remaining keys + String regex = null; + if (!(partVals.size() == table.getPartitionKeys().size() && firstStar == -1)) { + StringBuilder buf = new StringBuilder(".*"); + for (int i = Math.max(0, firstStar); + i < table.getPartitionKeys().size() && i < partVals.size(); i++) { + buf.append(HBaseUtils.KEY_SEPARATOR); + if ("*".equals(partVals.get(i))) { + buf.append("[^"); + buf.append(HBaseUtils.KEY_SEPARATOR); + buf.append("]+"); + } else { + buf.append(partVals.get(i)); + } + } + if (partVals.size() < table.getPartitionKeys().size()) { + buf.append(HBaseUtils.KEY_SEPARATOR); + buf.append(".*"); + } + regex = buf.toString(); } - keyPrefix = buildPartitionKey(dbName, tableName, partVals.subList(0, firstNull)); - StringBuilder regex = new StringBuilder(); - regex.append(dbName); - regex.append(':'); - regex.append(tableName); - for (String val : partVals) { - regex.append(HBaseUtils.KEY_SEPARATOR); - if (val == null) regex.append("[^" + HBaseUtils.KEY_SEPARATOR + "]+"); // Will this do - // what I want? - else regex.append(val); + + Filter filter = null; + if (regex != null) { + filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); } - Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, - new RegexStringComparator(regex.toString())); + if (LOG.isDebugEnabled()) { + LOG.debug("Scanning partitions with prefix <" + new String(keyPrefix) + "> and filter <" + + regex + ">"); + } List parts = scanPartitionsWithFilter(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions, filter); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java index 88c9f4b..0ab6551 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java @@ -161,7 +161,7 @@ public void part() throws IOException, TException { System.err.println("No such table: " + dbName + "." + tableName); return; } - String partName = HBaseStore.partName(table, partVals); + String partName = HBaseStore.buildExternalPartName(table, partVals); List stats = hrw.getPartitionStatistics(dbName, tableName, Arrays.asList(partName), Arrays.asList(partVals), colNames); if (stats == null) { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java index a4a88e4..0695dd2 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; @@ -212,7 +213,7 @@ public boolean dropTable(String dbName, String tableName) throws MetaException, return true; } catch (IOException e) { LOG.error("Unable to delete db" + e); - throw new MetaException("Unable to drop table " + tableName(dbName, tableName)); + throw new MetaException("Unable to drop table " + tableNameForErrorMsg(dbName, tableName)); } } @@ -221,7 +222,7 @@ public Table getTable(String dbName, String tableName) throws MetaException { try { Table table = getHBase().getTable(dbName, tableName); if (table == null) { - LOG.debug("Unable to find table " + tableName(dbName, tableName)); + LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName)); } return table; } catch (IOException e) { @@ -236,22 +237,18 @@ public boolean addPartition(Partition part) throws InvalidObjectException, MetaE getHBase().putPartition(part); return true; } catch (IOException e) { - // TODO NOt sure what i should throw here LOG.error("Unable to add partition", e); throw new MetaException("Unable to read from or write to hbase " + e.getMessage()); } } @Override - public boolean addPartitions(String dbName, String tblName, List parts) throws - InvalidObjectException, MetaException { + public boolean addPartitions(String dbName, String tblName, List parts) + throws InvalidObjectException, MetaException { try { - for (Partition part : parts) { - getHBase().putPartition(part); - } + getHBase().putPartitions(parts); return true; } catch (IOException e) { - // TODO NOt sure what i should throw here LOG.error("Unable to add partitions", e); throw new MetaException("Unable to read from or write to hbase " + e.getMessage()); } @@ -270,7 +267,7 @@ public Partition getPartition(String dbName, String tableName, List part Partition part = getHBase().getPartition(dbName, tableName, part_vals); if (part == null) { throw new NoSuchObjectException("Unable to find partition " + - partName(dbName, tableName, part_vals)); + partNameForErrorMsg(dbName, tableName, part_vals)); } return part; } catch (IOException e) { @@ -282,7 +279,12 @@ public Partition getPartition(String dbName, String tableName, List part @Override public boolean doesPartitionExist(String dbName, String tableName, List part_vals) throws MetaException, NoSuchObjectException { - throw new UnsupportedOperationException(); + try { + return getHBase().getPartition(dbName, tableName, part_vals) != null; + } catch (IOException e) { + LOG.error("Unable to get partition", e); + throw new MetaException("Error reading partition " + e.getMessage()); + } } @Override @@ -293,7 +295,8 @@ public boolean dropPartition(String dbName, String tableName, List part_ return true; } catch (IOException e) { LOG.error("Unable to delete db" + e); - throw new MetaException("Unable to drop partition " + partName(dbName, tableName, part_vals)); + throw new MetaException("Unable to drop partition " + partNameForErrorMsg(dbName, tableName, + part_vals)); } } @@ -315,8 +318,8 @@ public void alterTable(String dbname, String name, Table newTable) throws Invali try { getHBase().putTable(newTable); } catch (IOException e) { - LOG.error("Unable to alter table " + tableName(dbname, name), e); - throw new MetaException("Unable to alter table " + tableName(dbname, name)); + LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e); + throw new MetaException("Unable to alter table " + tableNameForErrorMsg(dbname, name)); } } @@ -352,6 +355,7 @@ public void alterTable(String dbname, String name, Table newTable) throws Invali @Override public List listTableNamesByFilter(String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { + // TODO needs to wait until we support pushing filters into HBase. throw new UnsupportedOperationException(); } @@ -364,7 +368,7 @@ public void alterTable(String dbname, String name, Table newTable) throws Invali List names = new ArrayList(parts.size()); Table table = getHBase().getTable(db_name, tbl_name); for (Partition p : parts) { - names.add(partName(table, p)); + names.add(buildExternalPartName(table, p)); } return names; } catch (IOException e) { @@ -376,20 +380,31 @@ public void alterTable(String dbname, String name, Table newTable) throws Invali @Override public List listPartitionNamesByFilter(String db_name, String tbl_name, String filter, short max_parts) throws MetaException { + // TODO needs to wait until we support pushing filters into HBase. throw new UnsupportedOperationException(); } @Override public void alterPartition(String db_name, String tbl_name, List part_vals, Partition new_part) throws InvalidObjectException, MetaException { - + try { + getHBase().putPartition(new_part); + } catch (IOException e) { + LOG.error("Unable to add partition", e); + throw new MetaException("Unable to read from or write to hbase " + e.getMessage()); + } } @Override public void alterPartitions(String db_name, String tbl_name, List> part_vals_list, List new_parts) throws InvalidObjectException, MetaException { - throw new UnsupportedOperationException(); + try { + getHBase().putPartitions(new_parts); + } catch (IOException e) { + LOG.error("Unable to add partition", e); + throw new MetaException("Unable to read from or write to hbase " + e.getMessage()); + } } @Override @@ -432,6 +447,7 @@ public void alterIndex(String dbname, String baseTblName, String name, Index new public List getPartitionsByFilter(String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { + // TODO - Needs to wait for ability to push filters into HBase throw new UnsupportedOperationException(); } @@ -644,36 +660,46 @@ public Role getRole(String roleName) throws NoSuchObjectException { @Override public Partition getPartitionWithAuth(String dbName, String tblName, List partVals, - String user_name, List group_names) throws - MetaException, NoSuchObjectException, InvalidObjectException { - Partition p = getPartition(dbName, tblName, partVals); - // TODO check that user is authorized to see these partitions - return p; + String user_name, List group_names) + throws MetaException, NoSuchObjectException, InvalidObjectException { + // We don't do authorization checks for partitions. + return getPartition(dbName, tblName, partVals); } @Override public List getPartitionsWithAuth(String dbName, String tblName, short maxParts, - String userName, List groupNames) throws - MetaException, NoSuchObjectException, InvalidObjectException { - List parts = getPartitions(dbName, tblName, maxParts); - // TODO check that user is authorized; - return parts; + String userName, List groupNames) + throws MetaException, NoSuchObjectException, InvalidObjectException { + // We don't do authorization checks for partitions. + return getPartitions(dbName, tblName, maxParts); } @Override public List listPartitionNamesPs(String db_name, String tbl_name, List part_vals, - short max_parts) throws MetaException, - NoSuchObjectException { - throw new UnsupportedOperationException(); + short max_parts) + throws MetaException, NoSuchObjectException { + List parts = + listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null); + List partNames = new ArrayList(parts.size()); + for (Partition part : parts) { + partNames.add(buildExternalPartName(db_name, tbl_name, part.getValues())); + } + return partNames; } @Override public List listPartitionsPsWithAuth(String db_name, String tbl_name, List part_vals, short max_parts, - String userName, List groupNames) throws - MetaException, InvalidObjectException, NoSuchObjectException { - throw new UnsupportedOperationException(); + String userName, List groupNames) + throws MetaException, NoSuchObjectException { + // We don't handle auth info with partitions + try { + return getHBase().scanPartitions(db_name, tbl_name, part_vals, max_parts); + } catch (IOException e) { + LOG.error("Unable to list partition names", e); + throw new MetaException("Failed to list part names, " + e.getMessage()); + } } @Override @@ -964,31 +990,37 @@ private HBaseReadWrite getHBase() { return hbase; } - private String tableName(String dbName, String tableName) { + // This is for building error messages only. It does not look up anything in the metastore. + private String tableNameForErrorMsg(String dbName, String tableName) { return dbName + "." + tableName; } - private String partName(String dbName, String tableName, List partVals) { - return tableName(dbName, tableName) + StringUtils.join(partVals, ':'); + // This is for building error messages only. It does not look up anything in the metastore as + // they may just throw another error. + private String partNameForErrorMsg(String dbName, String tableName, List partVals) { + return tableNameForErrorMsg(dbName, tableName) + "." + StringUtils.join(partVals, ':'); } - private String partName(Table table, Partition part) { - return partName(table, part.getValues()); + private String buildExternalPartName(Table table, Partition part) { + return buildExternalPartName(table, part.getValues()); } - static String partName(Table table, List partVals) { - List partCols = table.getPartitionKeys(); - StringBuilder builder = new StringBuilder(); - if (partCols.size() != partVals.size()) { - throw new RuntimeException("Woh bad, different number of partition cols and vals!"); - } - for (int i = 0; i < partCols.size(); i++) { - if (i != 0) builder.append('/'); - builder.append(partCols.get(i).getName()); - builder.append('='); - builder.append(partVals.get(i)); - } - return builder.toString(); + private String buildExternalPartName(String dbName, String tableName, List partVals) + throws MetaException { + return buildExternalPartName(getTable(dbName, tableName), partVals); + } + + /** + * Build a partition name for external use. Necessary since HBase itself doesn't store + * partition names. + * @param table table object + * @param partVals partition values. + * @return + */ + static String buildExternalPartName(Table table, List partVals) { + List partCols = new ArrayList(); + for (FieldSchema pc : table.getPartitionKeys()) partCols.add(pc.getName()); + return FileUtils.makePartName(partCols, partVals); } private List partNameToVals(String name) { diff --git metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java index 8cc2778..857e3f0 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java @@ -250,6 +250,55 @@ public void createPartition() throws Exception { Assert.assertEquals(tableName, p.getTableName()); Assert.assertEquals(1, p.getValuesSize()); Assert.assertEquals("fred", p.getValues().get(0)); + + Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals)); + Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob"))); + } + + @Test + public void alterPartition() throws Exception { + String dbName = "default"; + String tableName = "alterparttable"; + int startTime = (int)(System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "nocomment")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, + serde, null, null, emptyParameters); + List partCols = new ArrayList(); + partCols.add(new FieldSchema("pc", "string", "")); + Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + store.createTable(table); + + List vals = Arrays.asList("fred"); + StorageDescriptor psd = new StorageDescriptor(sd); + psd.setLocation("file:/tmp/pc=fred"); + Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd, + emptyParameters); + store.addPartition(part); + + part.setLastAccessTime(startTime + 10); + store.alterPartition(dbName, tableName, vals, part); + + Partition p = store.getPartition(dbName, tableName, vals); + Assert.assertEquals(1, p.getSd().getColsSize()); + Assert.assertEquals("col1", p.getSd().getCols().get(0).getName()); + Assert.assertEquals("int", p.getSd().getCols().get(0).getType()); + Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment()); + Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName()); + Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib()); + Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation()); + Assert.assertEquals("input", p.getSd().getInputFormat()); + Assert.assertEquals("output", p.getSd().getOutputFormat()); + Assert.assertEquals(dbName, p.getDbName()); + Assert.assertEquals(tableName, p.getTableName()); + Assert.assertEquals(1, p.getValuesSize()); + Assert.assertEquals("fred", p.getValues().get(0)); + Assert.assertEquals(startTime + 10, p.getLastAccessTime()); + + Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals)); + Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob"))); } @Test