diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 54306234bf..0212e076cd 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -338,8 +338,8 @@ public Table alterTable(String catName, String dbName, String name, Table newTab } @Override - public List getTables(String catName, String dbName, String pattern, TableType tableType) throws MetaException { - return objectStore.getTables(catName, dbName, pattern, tableType); + public List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException { + return objectStore.getTables(catName, dbName, pattern, tableType, limit); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 6c4896d912..fad96f4d6b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -348,6 +348,14 @@ private static HiveConf createHiveConf() { return (session == null) ? new HiveConf(Hive.class) : session.getConf(); } + public void setHMSClientCapabilities(String[] capabilities) { + HiveMetaStoreClient.setProcessorCapabilities(capabilities); + } + + public void setHMSClientIdentifier(final String id) { + HiveMetaStoreClient.setProcessorIdentifier(id); + } + private static boolean isCompatible(Hive db, HiveConf c, boolean isFastCheck) { if (isFastCheck) { return (db.metaStoreClient == null || db.metaStoreClient.isSameConfObj(c)) diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 7d948e448b..d2c9c1be4e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -3058,13 +3058,14 @@ public Table get_table(final String dbname, final String name) throws MetaExcept String database = req.getDatabase(); String pattern = req.getTableNamePattern(); List processorCapabilities = req.getProcessorCapabilities(); + int limit = req.getLimit(); String processorId = req.getProcessorIdentifier(); List tObjects = new ArrayList<>(); startTableFunction("get_tables_ext", catalog, database, pattern); Exception ex = null; try { - tables = getMS().getTables(catalog, database, pattern, null); + tables = getMS().getTables(catalog, database, pattern, null, limit); LOG.debug("get_tables_ext:getTables() returned " + tables.size()); tables = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook, catalog, database, tables); @@ -5469,7 +5470,7 @@ private void alter_table_core(String catName, String dbname, String name, Table List ret = null; Exception ex = null; try { - ret = getMS().getTables(catName, dbname, pattern, TableType.valueOf(tableType)); + ret = getMS().getTables(catName, dbname, pattern, TableType.valueOf(tableType), -1); } catch (MetaException e) { ex = e; throw e; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 45b89e0ebf..c11b3b8328 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -449,7 +449,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ * @param tableType Table type, or null if we want to get all tables * @return list of table names */ - public List getTables(String catName, String dbName, TableType tableType) + public List getTables(String catName, String dbName, TableType tableType, int limit) throws MetaException { String queryText = "SELECT " + TBLS + ".\"TBL_NAME\"" + " FROM " + TBLS + " " @@ -457,6 +457,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ + " WHERE " + DBS + ".\"NAME\" = ? AND " + DBS + ".\"CTLG_NAME\" = ? " + (tableType == null ? "" : "AND " + TBLS + ".\"TBL_TYPE\" = ? ") ; + List pms = new ArrayList<>(); pms.add(dbName); pms.add(catName); @@ -466,7 +467,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ Query queryParams = pm.newQuery("javax.jdo.query.SQL", queryText); return executeWithArray( - queryParams, pms.toArray(), queryText); + queryParams, pms.toArray(), queryText, limit); } /** @@ -753,10 +754,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - if (max != null) { - query.setRange(0, max.shortValue()); - } - List sqlResult = executeWithArray(query, params, queryText); + List sqlResult = executeWithArray(query, params, queryText, ((max == null) ? -1 : max.intValue())); long queryTime = doTrace ? System.nanoTime() : 0; MetastoreDirectSqlUtils.timingTrace(doTrace, queryText, start, queryTime); if (sqlResult.isEmpty()) { @@ -1879,7 +1877,12 @@ private String makeParams(int size) { @SuppressWarnings("unchecked") private T executeWithArray(Query query, Object[] params, String sql) throws MetaException { - return MetastoreDirectSqlUtils.executeWithArray(query, params, sql); + return executeWithArray(query, params, sql, -1); + } + + @SuppressWarnings("unchecked") + private T executeWithArray(Query query, Object[] params, String sql, int limit) throws MetaException { + return MetastoreDirectSqlUtils.executeWithArray(query, params, sql, limit); } /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java index 0d35fe459f..6a61c2a73e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDirectSqlUtils.java @@ -57,7 +57,14 @@ private MetastoreDirectSqlUtils() { } @SuppressWarnings("unchecked") static T executeWithArray(Query query, Object[] params, String sql) throws MetaException { + return (T)executeWithArray(query, params, sql, -1); + } + + @SuppressWarnings("unchecked") + static T executeWithArray(Query query, Object[] params, String sql, int limit) throws MetaException { try { + if (limit > 0) + query.setRange(0, limit); return (T)((params == null) ? query.execute() : query.executeWithArray(params)); } catch (Exception ex) { StringBuilder errorBuilder = new StringBuilder("Failed to execute [" + sql + "] with parameters ["); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 61019c6a1a..467168dc53 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1291,17 +1291,17 @@ public Table getTable(String catName, String dbName, String tableName, String wr @Override public List getTables(String catName, String dbName, String pattern) throws MetaException { - return getTables(catName, dbName, pattern, null); + return getTables(catName, dbName, pattern, null, -1); } @Override - public List getTables(String catName, String dbName, String pattern, TableType tableType) + public List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException { try { // We only support pattern matching via jdo since pattern matching in Java // might be different than the one used by the metastore backends return getTablesInternal(catName, dbName, pattern, tableType, - (pattern == null || pattern.equals(".*")), true); + (pattern == null || pattern.equals(".*")), true, limit); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } @@ -1398,7 +1398,7 @@ protected String describeResult() { } protected List getTablesInternal(String catName, String dbName, String pattern, - TableType tableType, boolean allowSql, boolean allowJdo) + TableType tableType, boolean allowSql, boolean allowJdo, int limit) throws MetaException, NoSuchObjectException { final String db_name = normalizeIdentifier(dbName); final String cat_name = normalizeIdentifier(catName); @@ -1406,19 +1406,19 @@ protected String describeResult() { @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { - return directSql.getTables(cat_name, db_name, tableType); + return directSql.getTables(cat_name, db_name, tableType, limit); } @Override protected List getJdoResult(GetHelper> ctx) throws MetaException, NoSuchObjectException { - return getTablesInternalViaJdo(cat_name, db_name, pattern, tableType); + return getTablesInternalViaJdo(cat_name, db_name, pattern, tableType, limit); } }.run(false); } private List getTablesInternalViaJdo(String catName, String dbName, String pattern, - TableType tableType) throws MetaException { + TableType tableType, int limit) throws MetaException { boolean commited = false; Query query = null; List tbls = null; @@ -1442,6 +1442,8 @@ protected String describeResult() { query = pm.newQuery(MTable.class, filterBuilder.toString()); query.setResult("tableName"); query.setOrdering("tableName ascending"); + if (limit > 0) + query.setRange(0, limit); Collection names = (Collection) query.executeWithArray(parameterVals.toArray(new String[0])); tbls = new ArrayList<>(names); commited = commitTransaction(); diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 6a93e264f1..c5e1a10869 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -392,10 +392,11 @@ void updateCreationMetadata(String catName, String dbname, String tablename, Cre * @param dbName database to search in * @param pattern pattern to match * @param tableType type of table to look for + * @param limit Maximum number of tables to return (undeterministic set) * @return list of table names, if any * @throws MetaException failure in querying the RDBMS */ - List getTables(String catName, String dbName, String pattern, TableType tableType) + List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException; /** diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index a5d0c046d4..1552ea0b8d 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1455,18 +1455,18 @@ public void updateCreationMetadata(String catName, String dbname, String tablena return rawStore.getTables(catName, dbName, pattern); } return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), - StringUtils.normalizeIdentifier(dbName), pattern, (short) -1); + StringUtils.normalizeIdentifier(dbName), pattern, -1); } @Override - public List getTables(String catName, String dbName, String pattern, TableType tableType) + public List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException { if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()|| !isCachedAllMetadata.get() || (canUseEvents && rawStore.isActiveTransaction())) { - return rawStore.getTables(catName, dbName, pattern, tableType); + return rawStore.getTables(catName, dbName, pattern, tableType, limit); } return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), - StringUtils.normalizeIdentifier(dbName), pattern, tableType); + StringUtils.normalizeIdentifier(dbName), pattern, tableType, limit); } @Override diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 05cf70b112..2c7354a881 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -1408,7 +1408,7 @@ public void alterTableAndStatsInCache(String catName, String dbName, String tblN } public List listCachedTableNames(String catName, String dbName, String pattern, - short maxTables) { + int maxTables) { List tableNames = new ArrayList<>(); try { cacheLock.readLock().lock(); @@ -1428,15 +1428,18 @@ public void alterTableAndStatsInCache(String catName, String dbName, String tblN } public List listCachedTableNames(String catName, String dbName, String pattern, - TableType tableType) { + TableType tableType, int limit) { List tableNames = new ArrayList<>(); try { cacheLock.readLock().lock(); + int count = 0; for (TableWrapper wrapper : tableCache.values()) { if (wrapper.sameDatabase(catName, dbName) && CacheUtils.matches(wrapper.getTable().getTableName(), pattern) - && wrapper.getTable().getTableType().equals(tableType.toString())) { + && wrapper.getTable().getTableType().equals(tableType.toString()) + && (limit == -1 || count < limit)) { tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); + count++; } } } finally { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index f202832e0b..da3c42a1d5 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -324,8 +324,8 @@ public void updateCreationMetadata(String catName, String dbname, String tablena } @Override - public List getTables(String catName, String dbName, String pattern, TableType tableType) throws MetaException { - return objectStore.getTables(catName, dbName, pattern, tableType); + public List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException { + return objectStore.getTables(catName, dbName, pattern, tableType, limit); } @Override diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 1a7ce04630..a018c503d1 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -308,7 +308,7 @@ public void updateCreationMetadata(String catName, String dbname, String tablena } @Override - public List getTables(String catName, String dbName, String pattern, TableType tableType) throws MetaException { + public List getTables(String catName, String dbName, String pattern, TableType tableType, int limit) throws MetaException { return Collections.emptyList(); } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index cbca6c3f30..0f2b8576e2 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -3908,6 +3908,34 @@ public void testGetTablesExt() throws Exception { assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size()); assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0); } + + extTables = client.getTablesExt(null, dbName, "*", requestedFields, (count - 3)); + LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields); + assertEquals("Return list size does not match expected size", (count - 3), extTables.size()); + for (ExtendedTableInfo tableInfo : extTables) { + assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size()); + assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0); + } + + extTables = client.getTablesExt(null, dbName, "*", requestedFields, -1); + LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields); + assertEquals("Return list size does not match expected size", count, extTables.size()); + + count = 300; + tProps.put("TBLNAME", "test_limit"); + tProps.put("TABLECOUNT", count); + tables = createTables(tProps); + assertEquals("Unexpected number of tables created", count, tables.size()); + + extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, count); + assertEquals("Unexpected number of tables returned", count, extTables.size()); + + extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, (count/2)); + assertEquals("Unexpected number of tables returned", (count/2), extTables.size()); + + extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, 1); + assertEquals("Unexpected number of tables returned", 1, extTables.size()); + } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); System.err.println("testGetTablesExt() failed."); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index 8caa929e7b..e30d4a8d1f 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -624,6 +624,10 @@ public void testGetTableByPattern() throws Exception { Assert.assertEquals(1, db1Tables.size()); db1Tables = cachedStore.getTables(DEFAULT_CATALOG_NAME, db1.getName(), ".*tbl1"); Assert.assertEquals(2, db1Tables.size()); + db1Tables = cachedStore.getTables(DEFAULT_CATALOG_NAME, db1.getName(), ".*tbl1", TableType.MANAGED_TABLE, 1); + Assert.assertEquals(1, db1Tables.size()); + db1Tables = cachedStore.getTables(DEFAULT_CATALOG_NAME, db1.getName(), ".*tbl1", TableType.MANAGED_TABLE, -1); + Assert.assertEquals(2, db1Tables.size()); cachedStore.shutdown(); }