diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java index 2d02707..58f165d 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HTableInterface; @@ -148,6 +147,38 @@ public void dropDb() throws Exception { } @Test + public void getAllDbs() throws Exception { + String[] dbNames = new String[3]; + for (int i = 0; i < dbNames.length; i++) { + dbNames[i] = "db" + i; + Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters); + store.createDatabase(db); + } + + List dbs = store.getAllDatabases(); + Assert.assertEquals(3, dbs.size()); + String[] namesFromStore = dbs.toArray(new String[3]); + Arrays.sort(namesFromStore); + Assert.assertArrayEquals(dbNames, namesFromStore); + } + + @Test + public void getDbsRegex() throws Exception { + String[] dbNames = new String[3]; + for (int i = 0; i < dbNames.length; i++) { + dbNames[i] = "db" + i; + Database db = new Database(dbNames[i], "no description", "file:///tmp", emptyParameters); + store.createDatabase(db); + } + + List dbs = store.getDatabases("db[12]"); + Assert.assertEquals(2, dbs.size()); + String[] namesFromStore = dbs.toArray(new String[2]); + Arrays.sort(namesFromStore); + Assert.assertArrayEquals(Arrays.copyOfRange(dbNames, 1, 3), namesFromStore); + } + + @Test public void createTable() throws Exception { int startTime = (int)(System.currentTimeMillis() / 1000); List cols = new ArrayList(); @@ -432,7 +463,7 @@ public void dropPartition() throws Exception { @Test public void createRole() throws Exception { - int now = (int)System.currentTimeMillis(); + int now = (int)System.currentTimeMillis()/1000; String roleName = "myrole"; store.addRole(roleName, "me"); @@ -444,14 +475,11 @@ public void createRole() throws Exception { @Test public void dropRole() throws Exception { - int now = (int)System.currentTimeMillis(); String roleName = "anotherrole"; store.addRole(roleName, "me"); Role r = store.getRole(roleName); Assert.assertEquals(roleName, r.getRoleName()); - Assert.assertEquals("me", r.getOwnerName()); - Assert.assertTrue(now <= r.getCreateTime()); store.removeRole(roleName); thrown.expect(NoSuchObjectException.class); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java index a475da0..9a4f657 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java @@ -259,6 +259,29 @@ Database getDb(String name) throws IOException { } /** + * Get a list of databases. + * @param regex Regular expression to use in searching for database names. It is expected to + * be a Java regular expression. If it is null then all databases will be returned. + * @return list of databases matching the regular expression. + * @throws IOException + */ + List scanDatabases(String regex) throws IOException { + Filter filter = null; + if (regex != null) { + filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex)); + } + Iterator iter = + scanWithFilter(DB_TABLE, null, CATALOG_CF, CATALOG_COL, filter); + List databases = new ArrayList(); + while (iter.hasNext()) { + DatabaseWritable db = new DatabaseWritable(); + HBaseUtils.deserialize(db, iter.next().getValue(CATALOG_CF, CATALOG_COL)); + databases.add(db.db); + } + return databases; + } + + /** * Store a database object * @param database database object to store * @throws IOException @@ -328,7 +351,7 @@ void putPartition(Partition partition) throws IOException { : new ArrayList(cached); } byte[] keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName); - List parts = scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL, -1); + List parts = scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, -1); partCache.put(dbName, tableName, parts, true); return maxPartitions < parts.size() ? parts.subList(0, maxPartitions) : parts; } @@ -352,7 +375,7 @@ void putPartition(Partition partition) throws IOException { byte[] keyPrefix; if (partVals == null || partVals.size() == 0) { keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName); - return scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); + return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); } int firstNull = 0; for (; firstNull < partVals.size(); firstNull++) { @@ -360,7 +383,7 @@ void putPartition(Partition partition) throws IOException { } if (firstNull == partVals.size()) { keyPrefix = buildPartitionKey(dbName, tableName, partVals); - return scanOnPrefix(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); + return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions); } keyPrefix = buildPartitionKey(dbName, tableName, partVals.subList(0, firstNull)); StringBuilder regex = new StringBuilder(); @@ -377,8 +400,8 @@ void putPartition(Partition partition) throws IOException { Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex.toString())); - List parts = scanOnPrefixWithFilter(PART_TABLE, keyPrefix, CATALOG_CF, CATALOG_COL, - maxPartitions, filter); + List parts = + scanPartitionsWithFilter(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions, filter); partCache.put(dbName, tableName, parts, false); return parts; } @@ -835,24 +858,18 @@ private void delete(String table, byte[] key, byte[] colFam, byte[] colName) thr htab.delete(d); } - private List scanOnPrefix(String table, byte[] keyPrefix, byte[] colFam, byte[] colName, - int maxResults) throws IOException { - return scanOnPrefixWithFilter(table, keyPrefix, colFam, colName, maxResults, null); + private List scanPartitions(byte[] keyPrefix, byte[] colFam, byte[] colName, + int maxResults) throws IOException { + return scanPartitionsWithFilter(keyPrefix, colFam, colName, maxResults, null); } - private List scanOnPrefixWithFilter(String table, byte[] keyPrefix, byte[] colFam, - byte[] colName, int maxResults, Filter filter) + private List scanPartitionsWithFilter(byte[] keyPrefix, byte[] colFam, byte[] colName, + int maxResults, Filter filter) throws IOException { - HTableInterface htab = getHTable(table); - byte[] stop = Arrays.copyOf(keyPrefix, keyPrefix.length); - stop[stop.length - 1]++; - Scan s = new Scan(keyPrefix, stop); - s.addColumn(colFam, colName); - if (filter != null) s.setFilter(filter); - ResultScanner scanner = htab.getScanner(s); + Iterator iter = + scanWithFilter(PART_TABLE, keyPrefix, colFam, colName, filter); List parts = new ArrayList(); int numToFetch = maxResults < 0 ? Integer.MAX_VALUE : maxResults; - Iterator iter = scanner.iterator(); for (int i = 0; i < numToFetch && iter.hasNext(); i++) { PartitionWritable p = new PartitionWritable(); HBaseUtils.deserialize(p, iter.next().getValue(colFam, colName)); @@ -861,6 +878,23 @@ private void delete(String table, byte[] key, byte[] colFam, byte[] colName) thr return parts; } + private Iterator scanWithFilter(String table, byte[] keyPrefix, byte[] colFam, + byte[] colName, Filter filter) throws IOException { + HTableInterface htab = getHTable(table); + Scan s; + if (keyPrefix == null) { + s = new Scan(); + } else { + byte[] stop = Arrays.copyOf(keyPrefix, keyPrefix.length); + stop[stop.length - 1]++; + s = new Scan(keyPrefix, stop); + } + s.addColumn(colFam, colName); + if (filter != null) s.setFilter(filter); + ResultScanner scanner = htab.getScanner(s); + return scanner.iterator(); + } + private HTableInterface getHTable(String table) throws IOException { HTableInterface htab = tables.get(table); if (htab == null) { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java index ed3525c..4eee858 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java @@ -148,17 +148,33 @@ public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaExc @Override public boolean alterDatabase(String dbname, Database db) throws NoSuchObjectException, MetaException { - throw new UnsupportedOperationException(); + // ObjectStore fetches the old db before updating it, but I can't see the possible value of + // that since the caller will have needed to call getDatabase to have the db object. + try { + getHBase().putDb(db); + return true; + } catch (IOException e) { + LOG.error("Unable to alter database ", e); + throw new MetaException("Unable to read from or write to hbase " + e.getMessage()); + } } @Override public List getDatabases(String pattern) throws MetaException { - throw new UnsupportedOperationException(); + try { + List dbs = getHBase().scanDatabases(pattern); + List dbNames = new ArrayList(dbs.size()); + for (Database db : dbs) dbNames.add(db.getName()); + return dbNames; + } catch (IOException e) { + LOG.error("Unable to get databases ", e); + throw new MetaException("Unable to get databases, " + e.getMessage()); + } } @Override public List getAllDatabases() throws MetaException { - throw new UnsupportedOperationException(); + return getDatabases(null); } @Override diff --git metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java index 76f0791..8cc2778 100644 --- metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java +++ metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java @@ -105,6 +105,20 @@ public void createDb() throws Exception { } @Test + public void alterDb() throws Exception { + String dbname = "mydb"; + Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters); + store.createDatabase(db); + db.setDescription("a description"); + store.alterDatabase(dbname, db); + + Database d = store.getDatabase(dbname); + Assert.assertEquals(dbname, d.getName()); + Assert.assertEquals("a description", d.getDescription()); + Assert.assertEquals("file:///tmp", d.getLocationUri()); + } + + @Test public void dropDb() throws Exception { String dbname = "anotherdb"; Database db = new Database(dbname, "no description", "file:///tmp", emptyParameters); @@ -352,7 +366,7 @@ public void dropPartition() throws Exception { @Test public void createRole() throws Exception { - int now = (int)System.currentTimeMillis(); + int now = (int)System.currentTimeMillis()/1000; String roleName = "myrole"; store.addRole(roleName, "me");