diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 748b56b0a268c1ec7dea022722478ec50889c016..c3f808d373483da0fcf8255a82b2f69e532eafbc 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hive.metastore; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import java.io.IOException; @@ -65,6 +67,7 @@ import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; +import org.apache.hadoop.hive.metastore.utils.FilterUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; @@ -127,6 +130,7 @@ private String tokenStrForm; private final boolean localMetaStore; private final MetaStoreFilterHook filterHook; + private final boolean isClientFilterEnabled; private final URIResolverHook uriResolverHook; private final int fileMetadataBatchSize; @@ -164,6 +168,7 @@ public HiveMetaStoreClient(Configuration conf, HiveMetaHookLoader hookLoader, Bo } version = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) ? TEST_VERSION : VERSION; filterHook = loadFilterHooks(); + isClientFilterEnabled = getIfClientFilterEnabled(); uriResolverHook = loadUriResolverHook(); fileMetadataBatchSize = MetastoreConf.getIntVar( conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX); @@ -276,6 +281,12 @@ public Void run() throws Exception { return null; } + private boolean getIfClientFilterEnabled() { + boolean isEnabled = MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_CLIENT_FILTER_ENABLED); + LOG.info("HMS client filtering is " + (isEnabled?"enabled.":"disabled.")); + + return isEnabled; + } private void resolveUris() throws MetaException { String thriftUris = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS); String serviceDiscoveryMode = MetastoreConf.getVar(conf, ConfVars.THRIFT_SERVICE_DISCOVERY_MODE); @@ -731,13 +742,15 @@ public void alterCatalog(String catalogName, Catalog newCatalog) throws TExcepti @Override public Catalog getCatalog(String catName) throws TException { GetCatalogResponse rsp = client.get_catalog(new GetCatalogRequest(catName)); - return rsp == null ? null : filterHook.filterCatalog(rsp.getCatalog()); + return rsp == null ? + null : FilterUtils.filterCatalogIfEnabled(isClientFilterEnabled, filterHook, rsp.getCatalog()); } @Override public List getCatalogs() throws TException { GetCatalogsResponse rsp = client.get_catalogs(); - return rsp == null ? null : filterHook.filterCatalogs(rsp.getNames()); + return rsp == null ? + null : FilterUtils.filterCatalogNamesIfEnabled(isClientFilterEnabled, filterHook, rsp.getNames()); } @Override @@ -808,7 +821,8 @@ public int add_partitions(List new_parts) throws TException { req.setCatName(part.isSetCatName() ? part.getCatName() : getDefaultCatalog(conf)); req.setNeedResult(needResults); AddPartitionsResult result = client.add_partitions_req(req); - return needResults ? filterHook.filterPartitions(result.getPartitions()) : null; + return needResults ? FilterUtils.filterPartitionsIfEnabled( + isClientFilterEnabled, filterHook, result.getPartitions()) : null; } @Override @@ -1619,8 +1633,9 @@ public boolean dropType(String type) throws NoSuchObjectException, MetaException @Override public List getDatabases(String catName, String databasePattern) throws TException { - return filterHook.filterDatabases(client.get_databases(prependCatalogToDbName( - catName, databasePattern, conf))); + List databases = client.get_databases(prependCatalogToDbName( + catName, databasePattern, conf)); + return FilterUtils.filterDbNamesIfEnabled(isClientFilterEnabled, filterHook, databases); } @Override @@ -1630,7 +1645,8 @@ public boolean dropType(String type) throws NoSuchObjectException, MetaException @Override public List getAllDatabases(String catName) throws TException { - return filterHook.filterDatabases(client.get_databases(prependCatalogToDbName(catName, null, conf))); + List databases = client.get_databases(prependCatalogToDbName(catName, null, conf)); + return FilterUtils.filterDbNamesIfEnabled(isClientFilterEnabled, filterHook, databases); } @Override @@ -1644,7 +1660,8 @@ public boolean dropType(String type) throws NoSuchObjectException, MetaException int max_parts) throws TException { List parts = client.get_partitions(prependCatalogToDbName(catName, db_name, conf), tbl_name, shrinkMaxtoShort(max_parts)); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions( + FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1655,8 +1672,10 @@ public PartitionSpecProxy listPartitionSpecs(String dbName, String tableName, in @Override public PartitionSpecProxy listPartitionSpecs(String catName, String dbName, String tableName, int maxParts) throws TException { - return PartitionSpecProxy.Factory.get(filterHook.filterPartitionSpecs( - client.get_partitions_pspec(prependCatalogToDbName(catName, dbName, conf), tableName, maxParts))); + List partitionSpecs = + client.get_partitions_pspec(prependCatalogToDbName(catName, dbName, conf), tableName, maxParts); + partitionSpecs = FilterUtils.filterPartitionSpecsIfEnabled(isClientFilterEnabled, filterHook, partitionSpecs); + return PartitionSpecProxy.Factory.get(partitionSpecs); } @Override @@ -1670,7 +1689,7 @@ public PartitionSpecProxy listPartitionSpecs(String catName, String dbName, Stri List part_vals, int max_parts) throws TException { List parts = client.get_partitions_ps(prependCatalogToDbName(catName, db_name, conf), tbl_name, part_vals, shrinkMaxtoShort(max_parts)); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1687,7 +1706,7 @@ public PartitionSpecProxy listPartitionSpecs(String catName, String dbName, Stri List groupNames) throws TException { List parts = client.get_partitions_with_auth(prependCatalogToDbName(catName, dbName, conf), tableName, shrinkMaxtoShort(maxParts), userName, groupNames); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1706,7 +1725,7 @@ public PartitionSpecProxy listPartitionSpecs(String catName, String dbName, Stri throws TException { List parts = client.get_partitions_ps_with_auth(prependCatalogToDbName(catName, dbName, conf), tableName, partialPvals, shrinkMaxtoShort(maxParts), userName, groupNames); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1720,7 +1739,7 @@ public PartitionSpecProxy listPartitionSpecs(String catName, String dbName, Stri String filter, int max_parts) throws TException { List parts =client.get_partitions_by_filter(prependCatalogToDbName( catName, db_name, conf), tbl_name, filter, shrinkMaxtoShort(max_parts)); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1734,9 +1753,11 @@ public PartitionSpecProxy listPartitionSpecsByFilter(String db_name, String tbl_ public PartitionSpecProxy listPartitionSpecsByFilter(String catName, String db_name, String tbl_name, String filter, int max_parts) throws TException { - return PartitionSpecProxy.Factory.get(filterHook.filterPartitionSpecs( + List partitionSpecs = client.get_part_specs_by_filter(prependCatalogToDbName(catName, db_name, conf), tbl_name, filter, - max_parts))); + max_parts); + return PartitionSpecProxy.Factory.get( + FilterUtils.filterPartitionSpecsIfEnabled(isClientFilterEnabled, filterHook, partitionSpecs)); } @Override @@ -1772,7 +1793,8 @@ public boolean listPartitionsByExpr(String catName, String db_name, String tbl_n throw new IncompatibleMetastoreException( "Metastore doesn't support listPartitionsByExpr: " + te.getMessage()); } - r.setPartitions(filterHook.filterPartitions(r.getPartitions())); + + r.setPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, r.getPartitions())); // TODO: in these methods, do we really need to deepcopy? deepCopyPartitions(r.getPartitions(), result); return !r.isSetHasUnknownPartitions() || r.isHasUnknownPartitions(); // Assume the worst. @@ -1786,7 +1808,7 @@ public Database getDatabase(String name) throws TException { @Override public Database getDatabase(String catalogName, String databaseName) throws TException { Database d = client.get_database(prependCatalogToDbName(catalogName, databaseName, conf)); - return deepCopy(filterHook.filterDatabase(d)); + return deepCopy(FilterUtils.filterDbIfEnabled(isClientFilterEnabled, filterHook, d)); } @Override @@ -1799,7 +1821,7 @@ public Partition getPartition(String db_name, String tbl_name, List part public Partition getPartition(String catName, String dbName, String tblName, List partVals) throws TException { Partition p = client.get_partition(prependCatalogToDbName(catName, dbName, conf), tblName, partVals); - return deepCopy(filterHook.filterPartition(p)); + return deepCopy(FilterUtils.filterPartitionIfEnabled(isClientFilterEnabled, filterHook, p)); } @Override @@ -1811,9 +1833,12 @@ public Partition getPartition(String catName, String dbName, String tblName, @Override public List getPartitionsByNames(String catName, String db_name, String tbl_name, List part_names) throws TException { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + checkDbAndTableFilters(catName, db_name, tbl_name); List parts = client.get_partitions_by_names(prependCatalogToDbName(catName, db_name, conf), tbl_name, part_names); - return deepCopyPartitions(filterHook.filterPartitions(parts)); + return deepCopyPartitions(FilterUtils.filterPartitionsIfEnabled(isClientFilterEnabled, filterHook, parts)); } @Override @@ -1839,7 +1864,7 @@ public Partition getPartitionWithAuthInfo(String catName, String dbName, String List groupNames) throws TException { Partition p = client.get_partition_with_auth(prependCatalogToDbName(catName, dbName, conf), tableName, pvals, userName, groupNames); - return deepCopy(filterHook.filterPartition(p)); + return deepCopy(FilterUtils.filterPartitionIfEnabled(isClientFilterEnabled, filterHook, p)); } @Override @@ -1853,7 +1878,7 @@ public Table getTable(String catName, String dbName, String tableName) throws TE req.setCatName(catName); req.setCapabilities(version); Table t = client.get_table_req(req).getTable(); - return deepCopy(filterHook.filterTable(t)); + return deepCopy(FilterUtils.filterTableIfEnabled(isClientFilterEnabled, filterHook, t)); } @Override @@ -1864,7 +1889,7 @@ public Table getTable(String catName, String dbName, String tableName, req.setCapabilities(version); req.setValidWriteIdList(validWriteIdList); Table t = client.get_table_req(req).getTable(); - return deepCopy(filterHook.filterTable(t)); + return deepCopy(FilterUtils.filterTableIfEnabled(isClientFilterEnabled, filterHook, t)); } @Override @@ -1881,7 +1906,7 @@ public Table getTable(String catName, String dbName, String tableName, req.setTblNames(tableNames); req.setCapabilities(version); List tabs = client.get_table_objects_by_name_req(req).getTables(); - return deepCopyTables(filterHook.filterTables(tabs)); + return deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled, filterHook, tabs)); } @Override @@ -1913,9 +1938,11 @@ public void updateCreationMetadata(String catName, String dbName, String tableNa @Override public List listTableNamesByFilter(String catName, String dbName, String filter, int maxTables) throws TException { - return filterHook.filterTableNames(catName, dbName, + List tableNames = client.get_table_names_by_filter(prependCatalogToDbName(catName, dbName, conf), filter, - shrinkMaxtoShort(maxTables))); + shrinkMaxtoShort(maxTables)); + return FilterUtils.filterTableNamesIfEnabled( + isClientFilterEnabled, filterHook, catName, dbName, tableNames); } /** @@ -1943,8 +1970,8 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE @Override public List getTables(String catName, String dbName, String tablePattern) throws TException { - return filterHook.filterTableNames(catName, dbName, - client.get_tables(prependCatalogToDbName(catName, dbName, conf), tablePattern)); + List tables = client.get_tables(prependCatalogToDbName(catName, dbName, conf), tablePattern); + return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tables); } @Override @@ -1960,9 +1987,10 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE @Override public List getTables(String catName, String dbName, String tablePattern, TableType tableType) throws TException { - return filterHook.filterTableNames(catName, dbName, + List tables = client.get_tables_by_type(prependCatalogToDbName(catName, dbName, conf), tablePattern, - tableType.toString())); + tableType.toString()); + return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tables); } @Override @@ -1974,8 +2002,9 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE public List getMaterializedViewsForRewriting(String catName, String dbname) throws MetaException { try { - return filterHook.filterTableNames(catName, dbname, - client.get_materialized_views_for_rewriting(prependCatalogToDbName(catName, dbname, conf))); + List views = + client.get_materialized_views_for_rewriting(prependCatalogToDbName(catName, dbname, conf)); + return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbname, views); } catch (Exception e) { MetaStoreUtils.logAndThrowMetaException(e); } @@ -1996,8 +2025,9 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE @Override public List getTableMeta(String catName, String dbPatterns, String tablePatterns, List tableTypes) throws TException { - return filterHook.filterTableMetas(catName,dbPatterns,client.get_table_meta(prependCatalogToDbName( - catName, dbPatterns, conf), tablePatterns, tableTypes)); + List tableMetas = client.get_table_meta(prependCatalogToDbName( + catName, dbPatterns, conf), tablePatterns, tableTypes); + return FilterUtils.filterTableMetasIfEnabled(isClientFilterEnabled, filterHook, catName,dbPatterns, tableMetas); } @Override @@ -2012,8 +2042,9 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE @Override public List getAllTables(String catName, String dbName) throws TException { - return filterHook.filterTableNames(catName, dbName, client.get_all_tables( - prependCatalogToDbName(catName, dbName, conf))); + List tableNames = client.get_all_tables( + prependCatalogToDbName(catName, dbName, conf)); + return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tableNames); } @Override @@ -2027,7 +2058,8 @@ public boolean tableExists(String catName, String dbName, String tableName) thro GetTableRequest req = new GetTableRequest(dbName, tableName); req.setCatName(catName); req.setCapabilities(version); - return filterHook.filterTable(client.get_table_req(req).getTable()) != null; + Table table = client.get_table_req(req).getTable(); + return FilterUtils.filterTableIfEnabled(isClientFilterEnabled, filterHook, table) != null; } catch (NoSuchObjectException e) { return false; } @@ -2042,8 +2074,11 @@ public boolean tableExists(String catName, String dbName, String tableName) thro @Override public List listPartitionNames(String catName, String dbName, String tableName, int maxParts) throws TException { - return filterHook.filterPartitionNames(catName, dbName, tableName, - client.get_partition_names(prependCatalogToDbName(catName, dbName, conf), tableName, shrinkMaxtoShort(maxParts))); + List partNames = + client.get_partition_names( + prependCatalogToDbName(catName, dbName, conf), tableName, shrinkMaxtoShort(maxParts)); + return FilterUtils.filterPartitionNamesIfEnabled( + isClientFilterEnabled, filterHook, catName, dbName, tableName, partNames); } @Override @@ -2055,9 +2090,10 @@ public boolean tableExists(String catName, String dbName, String tableName) thro @Override public List listPartitionNames(String catName, String db_name, String tbl_name, List part_vals, int max_parts) throws TException { - return filterHook.filterPartitionNames(catName, db_name, tbl_name, - client.get_partition_names_ps(prependCatalogToDbName(catName, db_name, conf), tbl_name, - part_vals, shrinkMaxtoShort(max_parts))); + List partNames = client.get_partition_names_ps(prependCatalogToDbName(catName, db_name, conf), tbl_name, + part_vals, shrinkMaxtoShort(max_parts)); + return FilterUtils.filterPartitionNamesIfEnabled( + isClientFilterEnabled, filterHook, catName, db_name, tbl_name, partNames); } @Override @@ -2384,7 +2420,7 @@ public Partition getPartition(String catName, String dbName, String tblName, Str throws TException { Partition p = client.get_partition_by_name(prependCatalogToDbName(catName, dbName, conf), tblName, name); - return deepCopy(filterHook.filterPartition(p)); + return deepCopy(FilterUtils.filterPartitionIfEnabled(isClientFilterEnabled, filterHook, p)); } public Partition appendPartitionByName(String dbName, String tableName, String partName) @@ -2419,6 +2455,22 @@ private HiveMetaHook getHook(Table tbl) throws MetaException { return hookLoader.getHook(tbl); } + /** + * This is a helper method to filter out a given database or a table name. This will improve + * performance when filtering partitions. If the db or table is filtered out, we don't need + * to even fetch the partitions. We can throw a NoSuchObjectException. + * @param catName the catalog name + * @param dbName the database name + * @param tblName the table name contained in the database + * @throws NoSuchObjectException if the database or table is filtered out + */ + private void checkDbAndTableFilters(final String catName, final String dbName, final String tblName) + throws NoSuchObjectException, MetaException { + + FilterUtils.checkDbAndTableFilters( + isClientFilterEnabled, filterHook, catName, dbName, tblName); + } + @Override public List partitionNameToVals(String name) throws MetaException, TException { return client.partition_name_to_vals(name); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index be1f8c78497fe3d0816ad3935ba07cd5ad379b08..75f0c0a356f3b894408aa54b9cce5220d47d7f26 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -216,7 +216,9 @@ public String toString() { ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL, ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL, ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, - ConfVars.FILE_METADATA_THREADS + ConfVars.FILE_METADATA_THREADS, + ConfVars.METASTORE_CLIENT_FILTER_ENABLED, + ConfVars.METASTORE_SERVER_FILTER_ENABLED }; /** @@ -657,6 +659,10 @@ public static ConfVars getMetaConf(String name) { "metadata being exported to the current user's home directory on HDFS."), METASTORE_MAX_EVENT_RESPONSE("metastore.max.event.response", "hive.metastore.max.event.response", 1000000, "The parameter will decide the maximum number of events that HMS will respond."), + METASTORE_CLIENT_FILTER_ENABLED("metastore.client.filter.enabled", "hive.metastore.client.filter.enabled", true, + "Enable filtering the metadata read results at HMS client. Default is true."), + METASTORE_SERVER_FILTER_ENABLED("metastore.server.filter.enabled", "hive.metastore.server.filter.enabled", false, + "Enable filtering the metadata read results at HMS server. Default is false."), MOVE_EXPORTED_METADATA_TO_TRASH("metastore.metadata.move.exported.metadata.to.trash", "hive.metadata.move.exported.metadata.to.trash", true, "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..ea1411368afbda5525175afaac75b905694d1c65 --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableMeta; + +/** + * Utilities common to Filtering operations. + */ +public class FilterUtils { + + public static Database filterDbIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + Database db) throws MetaException, NoSuchObjectException { + + if (isFilterEnabled) { + Database filteredDb = filterHook.filterDatabase(db); + + if (filteredDb == null) { + throw new NoSuchObjectException("DB " + db.getName() + " not found."); + } + } + + return db; + } + + public static List filterDbNamesIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + List dbNames) throws MetaException { + + if (isFilterEnabled) { + return filterHook.filterDatabases(dbNames); + } + + return dbNames; + } + + public static List filterTableNamesIfEnabled( + boolean isFilterEnabled, MetaStoreFilterHook filterHook, String catName, String dbName, + List tableNames) throws MetaException{ + + if (isFilterEnabled) { + return filterHook.filterTableNames(catName, dbName, tableNames); + } + + return tableNames; + } + + public static List

filterTablesIfEnabled( + boolean isFilterEnabled, MetaStoreFilterHook filterHook, List
tables) + throws MetaException{ + + if (isFilterEnabled) { + return filterHook.filterTables(tables); + } + + return tables; + } + + public static Table filterTableIfEnabled( + boolean isFilterEnabled, MetaStoreFilterHook filterHook, Table table) + throws MetaException, NoSuchObjectException { + if (isFilterEnabled) { + Table filteredTable = filterHook.filterTable(table); + + if (filteredTable == null) { + throw new NoSuchObjectException("Table " + table.getDbName() + "." + + table.getTableName() + " not found."); + } + } + + return table; + } + + public static List filterTableMetasIfEnabled( + boolean isFilterEnabled, MetaStoreFilterHook filterHook, + String catName, String dbName, List tableMetas) + throws MetaException, NoSuchObjectException { + if (tableMetas == null || tableMetas.isEmpty()) { + return tableMetas; + } + + if (isFilterEnabled) { + return filterHook.filterTableMetas( + catName, dbName, tableMetas); + } + + return tableMetas; + } + + public static Partition filterPartitionIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, Partition p) throws MetaException, NoSuchObjectException { + + if (isFilterEnabled) { + return filterHook.filterPartition(p); + } + + return p; + } + + public static List filterPartitionsIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, List partitions) throws MetaException { + + if (isFilterEnabled) { + return filterHook.filterPartitions(partitions); + } + + return partitions; + } + + public static List filterPartitionNamesIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + final String[] parsedDbName, final int catNameIndex, final int dbNameIndex, + final String tableName, List partitionNames) throws MetaException { + if (isFilterEnabled) { + return filterPartitionNamesIfEnabled(isFilterEnabled, filterHook, + parsedDbName, catNameIndex, parsedDbName[dbNameIndex], tableName, partitionNames); + } + + return partitionNames; + } + + public static List filterPartitionNamesIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + final String[] parsedDbName, final int catNameIndex, final String dbName, + final String tableName, List partitionNames) throws MetaException { + if (isFilterEnabled) { + return + filterPartitionNamesIfEnabled(isFilterEnabled, filterHook, parsedDbName[catNameIndex], + dbName, tableName, partitionNames); + } + + return partitionNames; + } + + public static List filterPartitionNamesIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + final String catName, final String dbName, + final String tableName, List partitionNames) throws MetaException { + if (isFilterEnabled) { + return + filterHook.filterPartitionNames(catName, + dbName, tableName, partitionNames); + } + + return partitionNames; + } + + public static List filterPartitionSpecsIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + List partitionSpecs) throws MetaException { + if (isFilterEnabled) { + return + filterHook.filterPartitionSpecs(partitionSpecs); + } + + return partitionSpecs; + } + + public static Catalog filterCatalogIfEnabled( + boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + Catalog catalog + ) throws MetaException, NoSuchObjectException { + if (isFilterEnabled) { + Catalog filteredCatalog = filterHook.filterCatalog(catalog); + + if (filteredCatalog == null) { + throw new NoSuchObjectException("Catalog " + catalog.getName() + " not found."); + } + } + + return catalog; + } + + public static List filterCatalogNamesIfEnabled( + boolean isFilterEnabled, MetaStoreFilterHook filterHook, + List catalogNames) throws MetaException{ + + if (isFilterEnabled) { + return filterHook.filterCatalogs(catalogNames); + } + + return catalogNames; + } + + + /** + * This is a helper method to filter out a given database or a table name. This will improve + * performance when filtering partitions. If the db or table is filtered out, we don't need + * to even fetch the partitions. We can throw a NoSuchObjectException. + * @param dbName the database name + * @param tblName the table name contained in the database + * @throws NoSuchObjectException if the database or table is filtered out + */ + public static void checkDbAndTableFilters(boolean isFilterEnabled, + MetaStoreFilterHook filterHook, + final String catName, final String dbName, final String tblName) + throws NoSuchObjectException, MetaException { + + List filteredDb = filterDbNamesIfEnabled(isFilterEnabled, filterHook, + Collections.singletonList(dbName)); + + if (filteredDb.isEmpty()) { + throw new NoSuchObjectException("Database " + dbName + " does not exist"); + } + + List filteredTable = + filterTableNamesIfEnabled(isFilterEnabled, filterHook, + catName, dbName, Collections.singletonList(tblName)); + if (filteredTable.isEmpty()) { + throw new NoSuchObjectException("Table " + tblName + " does not exist"); + } + } + +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a9398ae1e79404a15894aa42f451df5d18ed3e4c..b22e84a57becf18af63bb3464342de9cff56c2bf 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; +import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; @@ -30,6 +31,8 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -156,6 +159,7 @@ import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.utils.FilterUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.hive.metastore.utils.CommonCliOptions; @@ -506,6 +510,9 @@ public Configuration getHiveConf() { private List transactionalListeners; private List endFunctionListeners; private List initListeners; + private MetaStoreFilterHook filterHook; + private boolean isServerFilterEnabled = false; + private Pattern partitionValidationPattern; private final boolean isInTest; @@ -616,6 +623,94 @@ public void init() throws MetaException { } expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); fileMetadataManager = new FileMetadataManager(this.getMS(), conf); + + filterHook = loadFilterHooks(); + isServerFilterEnabled = getIfServerFilterenabled(); + } + + /** + * + * Filter is actually enabled only when the configured filter hook is configured, not default, and + * enabled in configuration + * @return + */ + private boolean getIfServerFilterenabled() { + boolean isEnabled = MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_SERVER_FILTER_ENABLED); + + if (!isEnabled) { + LOG.info("HMS server filtering is disabled by configuration"); + return false; + } + + String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK); + + if (isBlank(filterHookClassName)) { + LOG.info("HMS server filtering is disabled because no filter hook is configured"); + return false; + } + + if (filterHookClassName.trim().equalsIgnoreCase(DefaultMetaStoreFilterHookImpl.class.getName())) { + LOG.info("HMS server filtering is disabled because the filter hook is DefaultMetaStoreFilterHookImpl, which does no filtering"); + return false; + } + + LOG.info("HMS server filtering is enabled"); + return true; + } + + private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException { + String errorMsg = "Unable to load filter hook at HMS server. "; + try { + List filterHookList = MetaStoreServerUtils.getMetaStoreListeners( + MetaStoreFilterHook.class, conf, MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK)); + + if ((filterHookList == null) || (filterHookList.size() == 0)) { + errorMsg += "because no implementation class is specified"; + throw new IllegalStateException(errorMsg + errorMsg); + } + + if (filterHookList.size() > 1) { + LOG.info("More than one implementation classes are specified as HMS server filter." + + " Only use the first one: " + filterHookList.get(0).getClass().getName()); + } + + // only return the first implementation if multiple are specified. + return filterHookList.get(0); + } catch (MetaException e) { + throw new IllegalStateException(errorMsg + e.getMessage(), e); + } + } + + /** + * Check if user can access the table associated with the partition. If not, then throw exception + * so user cannot access partitions associated with this table + * We are not calling Pre event listener for authorization because it requires getting the + * table object from DB, more overhead. Instead ,we call filter hook to filter out table if user + * has no access. Filter hook only requires table name, not table object. That saves DB access for + * table object, and still achieve the same purpose: checking if user can access the specified + * table + * + * @param catName catalog name of the table + * @param dbName database name of the table + * @param tblName table name + * @throws NoSuchObjectException + * @throws MetaException + */ + private void authorizeTableForPartitionMetadata( + final String catName, final String dbName, final String tblName) + throws NoSuchObjectException, MetaException { + List filteredDb = FilterUtils.filterDbNamesIfEnabled( + isServerFilterEnabled, filterHook, Collections.singletonList(dbName)); + if (filteredDb.isEmpty()) { + throw new NoSuchObjectException("Database " + dbName + " does not exist"); + } + + List filteredTable = FilterUtils.filterTableNamesIfEnabled( + isServerFilterEnabled, filterHook, catName, + dbName, Collections.singletonList(tblName)); + if (filteredTable.isEmpty()) { + throw new NoSuchObjectException("Table " + tblName + " does not exist"); + } } private static String addPrefix(String s) { @@ -1160,7 +1255,7 @@ public GetCatalogResponse get_catalog(GetCatalogRequest rqst) ex = e; throw e; } finally { - endFunction("get_database", cat != null, ex); + endFunction("get_catalog", cat != null, ex); } } @@ -1677,8 +1772,10 @@ public void drop_database(final String dbName, final boolean deleteData, final b try { if (parsedDbNamed[DB_NAME] == null) { ret = getMS().getAllDatabases(parsedDbNamed[CAT_NAME]); + ret = FilterUtils.filterDbNamesIfEnabled(isServerFilterEnabled, filterHook, ret); } else { ret = getMS().getDatabases(parsedDbNamed[CAT_NAME], parsedDbNamed[DB_NAME]); + ret = FilterUtils.filterDbNamesIfEnabled(isServerFilterEnabled, filterHook, ret); } } catch (MetaException e) { ex = e; @@ -1694,6 +1791,7 @@ public void drop_database(final String dbName, final boolean deleteData, final b @Override public List get_all_databases() throws MetaException { + // get_databases filters results already. No need to filter here return get_databases(MetaStoreUtils.prependCatalogToDbName(null, null, conf)); } @@ -2901,7 +2999,7 @@ public Table get_table(final String dbname, final String name) throws MetaExcept NoSuchObjectException { String[] parsedDbName = parseDbName(dbname, conf); return getTableInternal( - parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, null, null); + parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, null, null); } @Override @@ -2929,6 +3027,7 @@ private Table getTableInternal(String catName, String dbname, String name, assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES, "insert-only tables", "get_table_req"); } + firePreEvent(new PreReadTableEvent(t, this)); } catch (MetaException | NoSuchObjectException e) { ex = e; @@ -2949,6 +3048,8 @@ private Table getTableInternal(String catName, String dbname, String name, Exception ex = null; try { t = getMS().getTableMeta(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames, tblTypes); + t = FilterUtils.filterTableMetasIfEnabled(isServerFilterEnabled, filterHook, + parsedDbName[CAT_NAME], parsedDbName[DB_NAME], t); } catch (Exception e) { ex = e; throw newMetaException(e); @@ -3014,8 +3115,8 @@ public Table get_table_core( @Override public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) throws TException { String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf); - return new GetTablesResult(getTableObjectsInternal(catName, - req.getDbName(), req.getTblNames(), req.getCapabilities())); + return new GetTablesResult(getTableObjectsInternal(catName, req.getDbName(), + req.getTblNames(), req.getCapabilities())); } private List
getTableObjectsInternal(String catName, String dbName, @@ -3068,6 +3169,8 @@ public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) throw "insert-only tables", "get_table_req"); } } + + FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tables); } catch (MetaException | InvalidOperationException | UnknownDBException e) { ex = e; throw e; @@ -3126,6 +3229,9 @@ private boolean doesClientHaveCapability(ClientCapabilities client, ClientCapabi throw new InvalidOperationException(filter + " cannot apply null filter"); } tables = getMS().listTableNamesByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], filter, maxTables); + tables = FilterUtils.filterTableNamesIfEnabled( + isServerFilterEnabled, filterHook, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tables); + } catch (MetaException | InvalidOperationException | UnknownDBException e) { ex = e; throw e; @@ -4526,8 +4632,12 @@ public Partition get_partition(final String db_name, final String tbl_name, Partition ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals); + ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; throwMetaException(e); @@ -4567,8 +4677,13 @@ public Partition get_partition_with_auth(final String db_name, Partition ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); + ret = getMS().getPartitionWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, user_name, group_names); + ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (InvalidObjectException e) { ex = e; throw new NoSuchObjectException(e.getMessage()); @@ -4592,8 +4707,14 @@ public Partition get_partition_with_auth(final String db_name, try { checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, NO_FILTER_STRING, max_parts); + + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); + ret = getMS().getPartitions(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; throwMetaException(e); @@ -4616,8 +4737,14 @@ public Partition get_partition_with_auth(final String db_name, try { checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, NO_FILTER_STRING, maxParts); + + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); + ret = getMS().getPartitionsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, maxParts, userName, groupNames); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (InvalidObjectException e) { ex = e; throw new NoSuchObjectException(e.getMessage()); @@ -4753,8 +4880,13 @@ private static boolean is_partition_spec_grouping_enabled(Table table) { List ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().listPartitionNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts); + ret = FilterUtils.filterPartitionNamesIfEnabled(isServerFilterEnabled, + filterHook, parsedDbName, CAT_NAME, DB_NAME, tbl_name, ret); } catch (MetaException e) { ex = e; throw e; @@ -5101,6 +5233,8 @@ private void alter_table_core(String catName, String dbname, String name, Table String[] parsedDbName = parseDbName(dbname, conf); try { ret = getMS().getTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern); + ret = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook, + parsedDbName[CAT_NAME], dbname, ret); } catch (MetaException e) { ex = e; throw e; @@ -5166,6 +5300,8 @@ private void alter_table_core(String catName, String dbname, String name, Table String[] parsedDbName = parseDbName(dbname, conf); try { ret = getMS().getAllTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); + ret = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook, + parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret); } catch (MetaException e) { ex = e; throw e; @@ -5418,6 +5554,7 @@ private Partition get_partition_by_name_core(final RawStore ms, final String cat throw new NoSuchObjectException(e.getMessage()); } Partition p = ms.getPartition(catName, db_name, tbl_name, partVals); + p = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, p); if (p == null) { throw new NoSuchObjectException(TableName.getQualified(catName, db_name, tbl_name) @@ -5438,7 +5575,9 @@ public Partition get_partition_by_name(final String db_name, final String tbl_na Exception ex = null; try { ret = get_partition_by_name_core(getMS(), parsedDbName[CAT_NAME], - parsedDbName[DB_NAME], tbl_name, part_name); } catch (Exception e) { + parsedDbName[DB_NAME], tbl_name, part_name); + ret = FilterUtils.filterPartitionIfEnabled(isServerFilterEnabled, filterHook, ret); + } catch (Exception e) { ex = e; rethrowException(e); } finally { @@ -5541,9 +5680,13 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n List ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); // Don't send the parsedDbName, as this method will parse itself. ret = get_partitions_ps_with_auth(db_name, tbl_name, part_vals, max_parts, null, null); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; rethrowException(e); @@ -5566,8 +5709,12 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n List ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().listPartitionsPsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, max_parts, userName, groupNames); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (InvalidObjectException e) { ex = e; throw new MetaException(e.getMessage()); @@ -5591,8 +5738,13 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n List ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().listPartitionNamesPs(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, max_parts); + ret = FilterUtils.filterPartitionNamesIfEnabled(isServerFilterEnabled, + filterHook, parsedDbName, CAT_NAME, DB_NAME, tbl_name, ret); } catch (Exception e) { ex = e; rethrowException(e); @@ -6035,8 +6187,14 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S try { checkLimitNumberOfPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, filter, maxParts); + + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); + ret = getMS().getPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, filter, maxParts); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; rethrowException(e); @@ -6170,8 +6328,12 @@ private int get_num_partitions_by_expr(final String catName, final String dbName List ret = null; Exception ex = null; try { + // For improved performance, we'll check if the said db and table are to be filtered out. + // If so, then we won't proceed with querying the partitions. + authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName); ret = getMS().getPartitionsByNames(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName, partNames); + ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret); } catch (Exception e) { ex = e; rethrowException(e); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 7dc69bc4e92875c8962dcd313b16f0f90ea8b057..4c9373985a2dfd67d2e945fd38b0ef537ec81a8c 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -247,8 +247,13 @@ public void testDummyFilterForPartition() throws Exception { } catch (NoSuchObjectException e) { // Excepted } - assertEquals(0, msc.getPartitionsByNames(DBNAME1, TAB2, - Lists.newArrayList("name=value1")).size()); + + try { + assertEquals(0, msc.getPartitionsByNames(DBNAME1, TAB2, + Lists.newArrayList("name=value1")).size()); + } catch (NoSuchObjectException e) { + // Excepted + } } } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreFilterHook.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreFilterHook.java new file mode 100644 index 0000000000000000000000000000000000000000..2c2e4d71324b304fc7c24431c77fe048fa7388ad --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreFilterHook.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; +import org.apache.hadoop.hive.metastore.api.TableMeta; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.util.StringUtils; +import org.junit.Test; + +import com.google.common.collect.Lists; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; + + +public class TestHiveMetastoreFilterHook { + public static class DummyMetaStoreFilterHookImpl implements MetaStoreFilterHook { + private static boolean blockResults = false; + + public DummyMetaStoreFilterHookImpl(Configuration conf) { + } + + @Override + public List filterDatabases(List dbList) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return dbList; + } + + @Override + public Database filterDatabase(Database dataBase) throws NoSuchObjectException { + if (blockResults) { + throw new NoSuchObjectException("Blocked access"); + } + return dataBase; + } + + @Override + public List filterTableNames(String catName, String dbName, List tableList) + throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return tableList; + } + + @Override + public Table filterTable(Table table) throws NoSuchObjectException { + if (blockResults) { + throw new NoSuchObjectException("Blocked access"); + } + return table; + } + + @Override + public List
filterTables(List
tableList) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return tableList; + } + + @Override + public List filterTableMetas(String catName, String dbName,List tableMetas) throws MetaException { + return tableMetas; + } + + @Override + public List filterPartitions(List partitionList) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return partitionList; + } + + @Override + public List filterPartitionSpecs( + List partitionSpecList) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return partitionSpecList; + } + + @Override + public Partition filterPartition(Partition partition) throws NoSuchObjectException { + if (blockResults) { + throw new NoSuchObjectException("Blocked access"); + } + return partition; + } + + @Override + public List filterPartitionNames(String catName, String dbName, String tblName, + List partitionNames) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return partitionNames; + } + } + + protected static HiveMetaStoreClient client; + protected static Configuration conf; + protected static Warehouse warehouse; + + private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100; + + private static String DBNAME1 = "testdb1"; + private static String DBNAME2 = "testdb2"; + private static final String TAB1 = "tab1"; + private static final String TAB2 = "tab2"; + + + protected HiveMetaStoreClient createClient(Configuration metaStoreConf) throws Exception { + try { + return new HiveMetaStoreClient(metaStoreConf); + } catch (Throwable e) { + System.err.println("Unable to open the metastore"); + System.err.println(StringUtils.stringifyException(e)); + throw new Exception(e); + } + } + + @BeforeClass + public static void setUp() throws Exception { + DummyMetaStoreFilterHookImpl.blockResults = true; + } + + @Before + public void setUpForTest() throws Exception { + + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setLongVar(conf, ConfVars.THRIFT_CONNECTION_RETRIES, 3); + MetastoreConf.setBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + MetastoreConf.setClass(conf, ConfVars.FILTER_HOOK, DummyMetaStoreFilterHookImpl.class, + MetaStoreFilterHook.class); + MetastoreConf.setBoolVar(conf, ConfVars.METRICS_ENABLED, true); + conf.set("hive.key1", "value1"); + conf.set("hive.key2", "http://www.example.com"); + conf.set("hive.key3", ""); + conf.set("hive.key4", "0"); + conf.set("datanucleus.autoCreateTables", "false"); + conf.set("hive.in.test", "true"); + + MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2); + MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, DEFAULT_LIMIT_PARTITION_REQUEST); + MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL, "no.such.class"); + MetaStoreTestUtils.setConfForStandloneMode(conf); + + warehouse = new Warehouse(conf); + } + + @After + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + } + + /** + * This is called in each test after the configuration is set in each test case + * @throws Exception + */ + protected void creatEnv(Configuration conf) throws Exception { + client = createClient(conf); + + client.dropDatabase(DBNAME1, true, true, true); + client.dropDatabase(DBNAME2, true, true, true); + Database db1 = new DatabaseBuilder() + .setName(DBNAME1) + .setCatalogName(Warehouse.DEFAULT_CATALOG_NAME) + .create(client, conf); + Database db2 = new DatabaseBuilder() + .setName(DBNAME2) + .setCatalogName(Warehouse.DEFAULT_CATALOG_NAME) + .create(client, conf); + new TableBuilder() + .setDbName(DBNAME1) + .setTableName(TAB1) + .addCol("id", "int") + .addCol("name", "string") + .create(client, conf); + Table tab2 = new TableBuilder() + .setDbName(DBNAME1) + .setTableName(TAB2) + .addCol("id", "int") + .addPartCol("name", "string") + .create(client, conf); + new PartitionBuilder() + .inTable(tab2) + .addValue("value1") + .addToTable(client, conf); + new PartitionBuilder() + .inTable(tab2) + .addValue("value2") + .addToTable(client, conf); + } + + /** + * The default configuration should be disable filtering at HMS server + * Disable the HMS client side filtering in order to see HMS server filtering behavior + * @throws Exception + */ + @Test + public void testHMSServerWithoutFilter() throws Exception { + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_CLIENT_FILTER_ENABLED, false); + DBNAME1 = "db_testHMSServerWithoutFilter_1"; + DBNAME2 = "db_testHMSServerWithoutFilter_2"; + creatEnv(conf); + + assertNotNull(client.getTable(DBNAME1, TAB1)); + assertEquals(2, client.getTables(DBNAME1, "*").size()); + assertEquals(2, client.getAllTables(DBNAME1).size()); + assertEquals(1, client.getTables(DBNAME1, TAB2).size()); + assertEquals(0, client.getAllTables(DBNAME2).size()); + + assertNotNull(client.getDatabase(DBNAME1)); + assertEquals(2, client.getDatabases("*testHMSServerWithoutFilter*").size()); + assertEquals(1, client.getDatabases(DBNAME1).size()); + + assertNotNull(client.getPartition(DBNAME1, TAB2, "name=value1")); + assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size()); + } + + /** + * Enable the HMS server side filtering + * Disable the HMS client side filtering in order to see HMS server filtering behavior + * @throws Exception + */ + @Test + public void testHMSServerWithFilter() throws Exception { + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_CLIENT_FILTER_ENABLED, false); + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_SERVER_FILTER_ENABLED, true); + DBNAME1 = "db_testHMSServerWithFilter_1"; + DBNAME2 = "db_testHMSServerWithFilter_2"; + creatEnv(conf); + + testFilterForDb(true); + testFilterForTables(true); + testFilterForPartition(); + } + + /** + * Disable filtering at HMS client + * By default, the HMS server side filtering is diabled, so we can see HMS client filtering behavior + * @throws Exception + */ + @Test + public void testHMSClientWithoutFilter() throws Exception { + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_CLIENT_FILTER_ENABLED, false); + DBNAME1 = "db_testHMSClientWithoutFilter_1"; + DBNAME2 = "db_testHMSClientWithoutFilter_2"; + creatEnv(conf); + + assertNotNull(client.getTable(DBNAME1, TAB1)); + assertEquals(2, client.getTables(DBNAME1, "*").size()); + assertEquals(2, client.getAllTables(DBNAME1).size()); + assertEquals(1, client.getTables(DBNAME1, TAB2).size()); + assertEquals(0, client.getAllTables(DBNAME2).size()); + + assertNotNull(client.getDatabase(DBNAME1)); + assertEquals(2, client.getDatabases("*testHMSClientWithoutFilter*").size()); + assertEquals(1, client.getDatabases(DBNAME1).size()); + + assertNotNull(client.getPartition(DBNAME1, TAB2, "name=value1")); + assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size()); + } + + /** + * By default, the HMS Client side filtering is enabled + * Disable the HMS server side filtering in order to see HMS client filtering behavior + * @throws Exception + */ + @Test + public void testHMSClientWithFilter() throws Exception { + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_SERVER_FILTER_ENABLED, false); + DBNAME1 = "db_testHMSClientWithFilter_1"; + DBNAME2 = "db_testHMSClientWithFilter_2"; + creatEnv(conf); + + testFilterForDb(false); + testFilterForTables(false); + testFilterForPartition(); + } + + protected void testFilterForDb(boolean filterAtServer) throws Exception { + + // Skip this call when testing filter hook at HMS server because HMS server calls authorization + // API for getDatabase(), and does not call filter hook + if (!filterAtServer) { + try { + assertNotNull(client.getDatabase(DBNAME1)); + fail("getDatabase() should fail with blocking mode"); + } catch (NoSuchObjectException e) { + // Excepted + } + } + + assertEquals(0, client.getDatabases("*").size()); + assertEquals(0, client.getAllDatabases().size()); + assertEquals(0, client.getDatabases(DBNAME1).size()); + } + + protected void testFilterForTables(boolean filterAtServer) throws Exception { + + // Skip this call when testing filter hook at HMS server because HMS server calls authorization + // API for getTable(), and does not call filter hook + if (!filterAtServer) { + try { + client.getTable(DBNAME1, TAB1); + fail("getTable() should fail with blocking mode"); + } catch (NoSuchObjectException e) { + // Excepted + } + } + + assertEquals(0, client.getTables(DBNAME1, "*").size()); + assertEquals(0, client.getAllTables(DBNAME1).size()); + assertEquals(0, client.getTables(DBNAME1, TAB2).size()); + } + + protected void testFilterForPartition() throws Exception { + try { + assertNotNull(client.getPartition(DBNAME1, TAB2, "name=value1")); + fail("getPartition() should fail with blocking mode"); + } catch (NoSuchObjectException e) { + // Excepted + } + + try { + client.getPartitionsByNames(DBNAME1, TAB2, + Lists.newArrayList("name=value1")).size(); + } catch (NoSuchObjectException e) { + // Excepted + } + } +}