Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java =================================================================== --- metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (revision 1145244) +++ metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (working copy) @@ -1528,4 +1528,186 @@ List databases = synchronizedClient.getAllDatabases(); assertEquals(1, databases.size()); } + + public void testTableFilter() throws Exception { + try { + String dbName = "testTableFilter"; + String owner1 = "testOwner1"; + String owner2 = "testOwner2"; + int lastAccessTime1 = 90; + int lastAccessTime2 = 30; + String tableName1 = "table1"; + String tableName2 = "table2"; + String tableName3 = "table3"; + + client.dropTable(dbName, tableName1); + client.dropTable(dbName, tableName2); + client.dropTable(dbName, tableName3); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + db.setDescription("Alter Partition Test database"); + client.createDatabase(db); + + Table table1 = createTableForTestFilter(dbName,tableName1, owner1, lastAccessTime1, true); + Table table2 = createTableForTestFilter(dbName,tableName2, owner2, lastAccessTime2, true); + Table table3 = createTableForTestFilter(dbName,tableName3, owner1, lastAccessTime2, false); + + List tableNames; + String filter; + //test owner + //owner like ".*Owner.*" and owner like "test.*" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_OWNER + + " like \".*Owner.*\" and " + + org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_OWNER + + " like \"test.*\""; + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(tableNames.size(), 3); + assert(tableNames.contains(table1.getTableName())); + assert(tableNames.contains(table2.getTableName())); + assert(tableNames.contains(table3.getTableName())); + + //owner = "testOwner1" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_OWNER + + " = \"testOwner1\""; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(2, tableNames.size()); + assert(tableNames.contains(table1.getTableName())); + assert(tableNames.contains(table3.getTableName())); + + //lastAccessTime < 90 + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_LAST_ACCESS + + " < 90"; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(2, tableNames.size()); + assert(tableNames.contains(table2.getTableName())); + assert(tableNames.contains(table3.getTableName())); + + //lastAccessTime > 90 + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_LAST_ACCESS + + " > 90"; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(0, tableNames.size()); + + //test params + //test_param_2 = "50" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_PARAMS + + "test_param_2 = \"50\""; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(2, tableNames.size()); + assert(tableNames.contains(table1.getTableName())); + assert(tableNames.contains(table2.getTableName())); + + //test_param_2 = "75" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_PARAMS + + "test_param_2 = \"75\""; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(0, tableNames.size()); + + //key_dne = "50" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_PARAMS + + "key_dne = \"50\""; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + assertEquals(0, tableNames.size()); + + //test_param_1 != "yellow" + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_PARAMS + + "test_param_1 <> \"yellow\""; + + tableNames = client.listTableNamesByFilter(dbName, filter, (short) 2); + assertEquals(2, tableNames.size()); + + //owner = "testOwner1" and (lastAccessTime = 30 or test_param_1 = "hi") + filter = org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_OWNER + + " = \"testOwner1\" and (" + + org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_LAST_ACCESS + + " = 30 or " + + org.apache.hadoop.hive.metastore.api.Constants.HIVE_FILTER_FIELD_PARAMS + + "test_param_1 = \"hi\")"; + tableNames = client.listTableNamesByFilter(dbName, filter, (short)-1); + + assertEquals(2, tableNames.size()); + assert(tableNames.contains(table1.getTableName())); + assert(tableNames.contains(table3.getTableName())); + + //Negative tests + Exception me = null; + try { + filter = "badKey = \"testOwner1\""; + tableNames = client.listTableNamesByFilter(dbName, filter, (short) -1); + } catch(MetaException e) { + me = e; + } + assertNotNull(me); + assertTrue("Bad filter key test", me.getMessage().contains( + "Invalid key name in filter")); + + client.dropTable(dbName, tableName1); + client.dropTable(dbName, tableName2); + client.dropTable(dbName, tableName3); + client.dropDatabase(dbName); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testTableFilter() failed."); + throw e; + } + } + + private Table createTableForTestFilter(String dbName, String tableName, String owner, int lastAccessTime, boolean hasSecondParam) throws Exception { + client.dropTable(dbName, tableName); + + ArrayList cols = new ArrayList(2); + cols.add(new FieldSchema("name", Constants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", Constants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(tableName); + tbl.setParameters(new HashMap()); + tbl.getParameters().put("test_param_1", "hi"); + if (hasSecondParam) { + tbl.getParameters().put("test_param_2", "50"); + } + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(cols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap()); + sd.getParameters().put("sd_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().getParameters() + .put(Constants.SERIALIZATION_FORMAT, "1"); + sd.setSortCols(new ArrayList()); + + tbl.setOwner(owner); + tbl.setLastAccessTime(lastAccessTime); + + tbl.setPartitionKeys(new ArrayList(2)); + tbl.getPartitionKeys().add( + new FieldSchema("ds", Constants.STRING_TYPE_NAME, "")); + tbl.getPartitionKeys().add( + new FieldSchema("hr", Constants.INT_TYPE_NAME, "")); + + client.createTable(tbl); + + if (isThriftClient) { + // the createTable() above does not update the location in the 'tbl' + // object when the client is a thrift client and the code below relies + // on the location being present in the 'tbl' object - so get the table + // from the metastore + tbl = client.getTable(dbName, tableName); + } + return tbl; + } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (working copy) @@ -135,6 +135,23 @@ public List getAllTables(String dbName) throws MetaException; + /** + * Gets a list of tables based on a filter string and filter type. + * @param dbName + * The name of the database from which you will retrieve the table names + * @param filterType + * The type of filter + * @param filter + * The filter string + * @param max_tables + * The maximum number of tables returned + * @return A list of table names that match the desired filter + * @throws MetaException + * @throws UnknownDBException + */ + public abstract List listTableNamesByFilter(String dbName, + String filter, short max_tables) throws MetaException, UnknownDBException; + public abstract List listPartitionNames(String db_name, String tbl_name, short max_parts) throws MetaException; Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (working copy) @@ -96,7 +96,48 @@ public List getAllTables(String dbName) throws MetaException, TException, UnknownDBException; + /** + * Get a list of table names that match a filter. + * The filter operators are LIKE, <, <=, >, >=, =, <> + * + * In the filter statement, values interpreted as strings must be enclosed in quotes, + * while values interpreted as integers should not be. Strings and integers are the only + * supported value types. + * + * The currently supported key names in the filter are: + * Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name + * and supports all filter operators + * Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times + * and supports all filter operators except LIKE + * Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values + * and only supports the filter operators = and <>. + * Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement. + * For example, to filter on parameter keys called "retention", the key name in the filter + * statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention" + * Also, = and <> only work for keys that exist in the tables. + * E.g., filtering on tables where key1 <> value will only + * return tables that have a value for the parameter key1. + * Some example filter statements include: + * filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " + + * Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0"; + * filter = Constants.HIVE_FILTER_FIELD_OWNER + " = \"test_user\" and (" + + * Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " + + * Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\")" + * + * @param dbName + * The name of the database from which you will retrieve the table names + * @param filterType + * The type of filter + * @param filter + * The filter string + * @param max_tables + * The maximum number of tables returned + * @return A list of table names that match the desired filter + */ + public List listTableNamesByFilter(String dbName, String filter, short maxTables) + throws MetaException, TException, InvalidOperationException, UnknownDBException; + /** * Drop the table. * Index: metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (working copy) @@ -25,9 +25,9 @@ import org.antlr.runtime.CharStream; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Constants; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.serde.Constants; /** * The Class representing the filter as a binary tree. The tree has TreeNode's @@ -103,7 +103,20 @@ this.rhs = rhs; } - public String generateJDOFilter(Table table, Map params) + /** + * Generates a JDO filter statement + * @param table + * The table on which the filter is applied. If table is not null, + * then this method generates a JDO statement to get all partitions + * of the table that match the filter. + * If table is null, then this method generates a JDO statement to get all + * tables that match the filter. + * @param params + * A map of parameter key to values for the filter statement. + * @return a JDO filter statement + * @throws MetaException + */ + public String generateJDOFilter(Table table, Map params) throws MetaException { StringBuilder filterBuffer = new StringBuilder(); @@ -125,7 +138,6 @@ return filterBuffer.toString(); } - } /** @@ -134,12 +146,83 @@ public static class LeafNode extends TreeNode { public String keyName; public Operator operator; - public String value; + public Object value; public boolean isReverseOrder = false; private static final String PARAM_PREFIX = "hive_filter_param_"; @Override - public String generateJDOFilter(Table table, Map params) + public String generateJDOFilter(Table table, + Map params) + throws MetaException { + if (table != null) { + return generateJDOFilterOverPartitions(table, params); + } else { + return generateJDOFilterOverTables(params); + } + } + + private String generateJDOFilterOverTables(Map params) + throws MetaException { + if (keyName.equals(Constants.HIVE_FILTER_FIELD_OWNER)) { + keyName = "this.owner"; + } else if (keyName.equals(Constants.HIVE_FILTER_FIELD_LAST_ACCESS)) { + //lastAccessTime expects an integer, so we cannot use the "like operator" + if (operator == Operator.LIKE) { + throw new MetaException("Like is not supported for HIVE_FILTER_FIELD_LAST_ACCESS"); + } + keyName = "this.lastAccessTime"; + } else if (keyName.startsWith(Constants.HIVE_FILTER_FIELD_PARAMS)) { + //can only support "=" and "<>" for now, because our JDO lib is buggy when + // using objects from map.get() + if (!(operator == Operator.EQUALS || operator == Operator.NOTEQUALS)) { + throw new MetaException("Only = and <> are supported " + + "opreators for HIVE_FILTER_FIELD_PARAMS"); + } + String paramKeyName = keyName.substring(Constants.HIVE_FILTER_FIELD_PARAMS.length()); + keyName = "this.parameters.get(\"" + paramKeyName + "\")"; + //value is persisted as a string in the db, so make sure it's a string here + // in case we get an integer. + value = value.toString(); + } else { + throw new MetaException("Invalid key name in filter. " + + "Use constants from org.apache.hadoop.hive.metastore.api"); + } + return generateJDOFilterGeneral(params); + } + + /** + * Generates a general filter. Given a map of , + * generates a statement of the form: + * key1 operator value2 (&& | || ) key2 operator value2 ... + * + * Currently supported types for value are String and Integer. + * The LIKE operator for Integers is unsupported. + */ + private String generateJDOFilterGeneral(Map params) + throws MetaException { + String paramName = PARAM_PREFIX + params.size(); + params.put(paramName, value); + String filter; + + if (isReverseOrder) { + if (operator == Operator.LIKE) { + throw new MetaException( + "Value should be on the RHS for LIKE operator : " + + "Key <" + keyName + ">"); + } else { + filter = paramName + " " + operator.getJdoOp() + " " + keyName; + } + } else { + if (operator == Operator.LIKE) { + filter = " " + keyName + "." + operator.getJdoOp() + "(" + paramName + ") "; + } else { + filter = " " + keyName + " " + operator.getJdoOp() + " " + paramName; + } + } + return filter; + } + + private String generateJDOFilterOverPartitions(Table table, Map params) throws MetaException { int partitionColumnCount = table.getPartitionKeys().size(); @@ -159,14 +242,22 @@ "> is not a partitioning key for the table"); } + //Can only support partitions whose types are string if( ! table.getPartitionKeys().get(partitionColumnIndex). - getType().equals(Constants.STRING_TYPE_NAME) ) { + getType().equals(org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME) ) { throw new MetaException ("Filtering is supported only on partition keys of type string"); } + String valueParam = null; + try { + valueParam = (String) value; + } catch (ClassCastException e) { + throw new MetaException("Filtering is supported only on partition keys of type string"); + } + String paramName = PARAM_PREFIX + params.size(); - params.put(paramName, value); + params.put(paramName, valueParam); String filter; String keyEqual = FileUtils.escapePathName(keyName) + "="; @@ -187,9 +278,8 @@ throw new MetaException( "Value should be on the RHS for LIKE operator : " + "Key <" + keyName + ">"); - } - else if (operator == Operator.EQUALS) { - filter = makeFilterForEquals(keyName, value, paramName, params, + } else if (operator == Operator.EQUALS) { + filter = makeFilterForEquals(keyName, valueParam, paramName, params, partitionColumnIndex, partitionColumnCount); } else { filter = paramName + @@ -201,7 +291,7 @@ filter = " " + valString + "." + operator.getJdoOp() + "(" + paramName + ") "; } else if (operator == Operator.EQUALS) { - filter = makeFilterForEquals(keyName, value, paramName, params, + filter = makeFilterForEquals(keyName, valueParam, paramName, params, partitionColumnIndex, partitionColumnCount); } else { filter = " " + valString + " " @@ -231,7 +321,7 @@ * @throws MetaException */ private static String makeFilterForEquals(String keyName, String value, - String paramName, Map params, int keyPos, int keyCount) + String paramName, Map params, int keyPos, int keyCount) throws MetaException { Map partKeyToVal = new HashMap(); partKeyToVal.put(keyName, value); @@ -308,7 +398,7 @@ * @throws MetaException */ public String generateJDOFilter(Table table, - Map params) throws MetaException { + Map params) throws MetaException { if( root == null ) { return ""; } Index: metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g (working copy) @@ -63,17 +63,26 @@ operatorExpression @init { boolean isReverseOrder = false; + Object val = null; } : ( - (key = Identifier op = operator value = StringLiteral) - | - (value = StringLiteral op = operator key = Identifier) { isReverseOrder = true; } + ( + (key = Identifier op = operator value = StringLiteral) + | + (value = StringLiteral op = operator key = Identifier) { isReverseOrder = true; } + ) { val = TrimQuotes(value.getText()); } + | + ( + (key = Identifier op = operator value = IntLiteral) + | + (value = IntLiteral op = operator key = Identifier) { isReverseOrder = true; } + ) { val = Integer.parseInt(value.getText()); } ) { LeafNode node = new LeafNode(); node.keyName = key.getText(); - node.value = TrimQuotes(value.getText()); + node.value = val; node.operator = op; node.isReverseOrder = isReverseOrder; @@ -121,10 +130,15 @@ ) ; + +IntLiteral + : + (Digit)+ + ; + Identifier : (Letter | Digit) (Letter | Digit | '_')* ; WS : (' '|'\r'|'\t'|'\n')+ { skip(); } ; - Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (working copy) @@ -697,6 +697,12 @@ return deepCopyTables(client.get_table_objects_by_name(dbName, tableNames)); } + /** {@inheritDoc} */ + public List listTableNamesByFilter(String dbName, String filter, short maxTables) + throws MetaException, TException, InvalidOperationException, UnknownDBException { + return client.get_table_names_by_filter(dbName, filter, maxTables); + } + /** * @param name * @return the type Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -1229,6 +1229,40 @@ return tables; } + @Override + public List get_table_names_by_filter( + final String dbName, final String filter, final short maxTables) + throws MetaException, InvalidOperationException, UnknownDBException { + List tables = new ArrayList(); + startFunction("get_table_names_by_filter", ": db = " + dbName + ", filter = " + filter); + try { + tables = executeWithRetry(new Command>() { + @Override + public List run(RawStore ms) throws Exception { + if (dbName == null || dbName.isEmpty()) { + throw new UnknownDBException("DB name is null or empty"); + } + if (filter == null) { + throw new InvalidOperationException(filter + " cannot apply null filter"); + } + List tables = ms.listTableNamesByFilter(dbName, filter, maxTables); + return tables; + } + }); + } catch (MetaException e) { + throw e; + } catch (InvalidOperationException e) { + throw e; + } catch (UnknownDBException e) { + throw e; + } catch (Exception e) { + throw new MetaException(e.toString()); + } finally { + endFunction("get_table_names_by_filter"); + } + return tables; + } + public boolean set_table_parameters(String dbname, String name, Map params) throws NoSuchObjectException, MetaException { endFunction(startTableFunction("set_table_parameters", dbname, name)); Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1145244) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -1483,31 +1483,49 @@ return parts; } - private String makeQueryFilterString(MTable mtable, String filter, - Map params) - throws MetaException { - StringBuilder queryBuilder = new StringBuilder( - "table.tableName == t1 && table.database.name == t2"); + private FilterParser getFilterParser(String filter) throws MetaException { + CharStream cs = new ANTLRNoCaseStringStream(filter); + FilterLexer lexer = new FilterLexer(cs); - if( filter != null && filter.length() > 0) { + CommonTokenStream tokens = new CommonTokenStream(); + tokens.setTokenSource (lexer); - Table table = convertToTable(mtable); + FilterParser parser = new FilterParser(tokens); - CharStream cs = new ANTLRNoCaseStringStream(filter); - FilterLexer lexer = new FilterLexer(cs); + try { + parser.filter(); + } catch(RecognitionException re) { + throw new MetaException("Error parsing partition filter : " + re); + } + return parser; + } - CommonTokenStream tokens = new CommonTokenStream(); - tokens.setTokenSource (lexer); + /** + * Makes a JDO query filter string + * if mtable is not null, generates the query to filter over partitions in a table. + * if mtable is null, generates the query to filter over tables in a database + */ + private String makeQueryFilterString(MTable mtable, String filter, + Map params) + throws MetaException { - FilterParser parser = new FilterParser(tokens); + StringBuilder queryBuilder = new StringBuilder(); + if (mtable != null) { + queryBuilder.append("table.tableName == t1 && table.database.name == t2"); + } else { + queryBuilder.append("database.name == dbName"); + } - try { - parser.filter(); - } catch(RecognitionException re) { - throw new MetaException("Error parsing partition filter : " + re); + if (filter != null && filter.length() > 0) { + FilterParser parser = getFilterParser(filter); + String jdoFilter; + + if (mtable != null) { + Table table = convertToTable(mtable); + jdoFilter = parser.tree.generateJDOFilter(table, params); + } else { + jdoFilter = parser.tree.generateJDOFilter(null, params); } - - String jdoFilter = parser.tree.generateJDOFilter(table, params); LOG.debug("jdoFilter = " + jdoFilter); if( jdoFilter.trim().length() > 0 ) { @@ -1519,15 +1537,33 @@ return queryBuilder.toString(); } + private String makeTableQueryFilterString(String filter, + Map params) + throws MetaException { + return makeQueryFilterString(null, filter, params); + } + private String makeParameterDeclarationString(Map params) { //Create the parameter declaration string StringBuilder paramDecl = new StringBuilder(); - for(String key : params.keySet() ) { - paramDecl.append(", java.lang.String " + key); + for (String key : params.keySet()) { + paramDecl.append(", java.lang.String " + key); } return paramDecl.toString(); } + private String makeParameterDeclarationStringObj(Map params) { + //Create the parameter declaration string + StringBuilder paramDecl = new StringBuilder(); + for (Entry entry : params.entrySet()) { + paramDecl.append(", "); + paramDecl.append(entry.getValue().getClass().getName()); + paramDecl.append(" "); + paramDecl.append(entry.getKey()); + } + return paramDecl.toString(); + } + private List listMPartitionsByFilter(String dbName, String tableName, String filter, short maxParts) throws MetaException, NoSuchObjectException{ boolean success = false; @@ -1543,7 +1579,7 @@ throw new NoSuchObjectException("Specified database/table does not exist : " + dbName + "." + tableName); } - Map params = new HashMap(); + Map params = new HashMap(); String queryFilterString = makeQueryFilterString(mtable, filter, params); @@ -1561,7 +1597,7 @@ params.put("t1", tableName.trim()); params.put("t2", dbName.trim()); - String parameterDeclaration = makeParameterDeclarationString(params); + String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); @@ -1580,6 +1616,52 @@ } @Override + public List listTableNamesByFilter(String dbName, String filter, short maxTables) + throws MetaException { + boolean success = false; + List tableNames = new ArrayList(); + try { + openTransaction(); + LOG.debug("Executing listTableNamesByFilter"); + dbName = dbName.toLowerCase().trim(); + Map params = new HashMap(); + String queryFilterString = makeTableQueryFilterString(filter, params); + Query query = pm.newQuery(MTable.class); + query.declareImports("import java.lang.String"); + query.setResult("tableName"); + query.setResultClass(java.lang.String.class); + if (maxTables >= 0) { + query.setRange(0, maxTables); + } + LOG.debug("filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString); + params.put("dbName", dbName); + for (Entry entry : params.entrySet()) { + LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() + + " class: " + entry.getValue().getClass().getName()); + } + String parameterDeclaration = makeParameterDeclarationStringObj(params); + query.declareParameters(parameterDeclaration); + query.setFilter(queryFilterString); + Collection names = (Collection) query.executeWithMap(params); + //have to emulate "distinct", otherwise tables with the same name may be returned + Set tableNamesSet = new HashSet(); + for (Iterator i = names.iterator(); i.hasNext();) { + tableNamesSet.add((String) i.next()); + } + tableNames = new ArrayList(tableNamesSet); + LOG.debug("Done executing query for listTableNamesByFilter"); + success = commitTransaction(); + LOG.debug("Done retrieving all objects for listTableNamesByFilter"); + + } finally { + if (!success) { + rollbackTransaction(); + } + } + return tableNames; + } + + @Override public List listPartitionNamesByFilter(String dbName, String tableName, String filter, short maxParts) throws MetaException { boolean success = false; @@ -1596,7 +1678,7 @@ // table or db does not exist, we return an empty list return partNames; } - Map params = new HashMap(); + Map params = new HashMap(); String queryFilterString = makeQueryFilterString(mtable, filter, params); Query query = pm.newQuery( @@ -1615,7 +1697,7 @@ params.put("t1", tableName.trim()); params.put("t2", dbName.trim()); - String parameterDeclaration = makeParameterDeclarationString(params); + String parameterDeclaration = makeParameterDeclarationStringObj(params); query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); query.setResult("partitionName"); Index: metastore/if/hive_metastore.thrift =================================================================== --- metastore/if/hive_metastore.thrift (revision 1145244) +++ metastore/if/hive_metastore.thrift (working copy) @@ -43,6 +43,10 @@ GROUP = 3, } +const string HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__" +const string HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__" +const string HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__" + enum PartitionEventType { LOAD_DONE = 1, } @@ -258,9 +262,46 @@ Table get_table(1:string dbname, 2:string tbl_name) throws (1:MetaException o1, 2:NoSuchObjectException o2) - list get_table_objects_by_name(1:string dbname, 2:list tbl_names) - throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) - + list
get_table_objects_by_name(1:string dbname, 2:list tbl_names) + throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) + + // Get a list of table names that match a filter. + // The filter operators are LIKE, <, <=, >, >=, =, <> + // + // In the filter statement, values interpreted as strings must be enclosed in quotes, + // while values interpreted as integers should not be. Strings and integers are the only + // supported value types. + // + // The currently supported key names in the filter are: + // Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name + // and supports all filter operators + // Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times + // and supports all filter operators except LIKE + // Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values + // and only supports the filter operators = and <>. + // Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement. + // For example, to filter on parameter keys called "retention", the key name in the filter + // statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention" + // Also, = and <> only work for keys that exist + // in the tables. E.g., if you are looking for tables where key1 <> value, it will only + // look at tables that have a value for the parameter key1. + // Some example filter statements include: + // filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " + + // Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0"; + // filter = Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " + + // Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\"" + // @param dbName + // The name of the database from which you will retrieve the table names + // @param filterType + // The type of filter + // @param filter + // The filter string + // @param max_tables + // The maximum number of tables returned + // @return A list of table names that match the desired filter + list get_table_names_by_filter(1:string dbname, 2:string filter, 3:i16 max_tables=-1) + throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) + // alter table applies to only future partitions not for existing partitions // * See notes on DDL_TIME void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)