commit 0deae2ef376b399ca91fafe3038b0edff1979619 Author: Vihang Karajgaonkar Date: Wed Aug 22 20:29:23 2018 -0700 HIVE-20307 : Add support for filterspec to the getPartitions with projection API diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index d59d5d807a26378a430e683533e53d0831cf9514..c3e1e8e88c69d7713e16c7061ce8cf73a0d5e833 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -19,6 +19,8 @@ package org.apache.hive.hcatalog.listener; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; +import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.Catalog; @@ -413,12 +415,10 @@ public Partition alterPartition(String catName, String dbName, String tblName, L } @Override - public List getPartitionSpecsByFilterAndProjection(String catalog, - String dbName, String tblName, - List fieldList, String includeParamKeyPattern, - String excludeParamKeyPattern) throws MetaException, NoSuchObjectException { - return objectStore.getPartitionSpecsByFilterAndProjection(catalog, dbName, tblName, fieldList, - includeParamKeyPattern, excludeParamKeyPattern); + public List getPartitionSpecsByFilterAndProjection(Table table, + GetPartitionsProjectionSpec projectionSpec, GetPartitionsFilterSpec filterSpec) + throws MetaException, NoSuchObjectException { + return objectStore.getPartitionSpecsByFilterAndProjection(table, projectionSpec, filterSpec); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index a2b57fb646899c54b63be14a8cde9b8644a973aa..dd23d7db3e70c9540e48c42eb7b9a33ed775cea6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -948,7 +948,7 @@ private Partition getPartition(String partName) { * */ private List getPartitions(List partialPartVals) throws MetaException { - String partNameMatcher = MetaStoreUtils.makePartNameMatcher(tTable, partialPartVals); + String partNameMatcher = MetaStoreUtils.makePartNameMatcher(tTable, partialPartVals, ".*"); List matchedPartitions = new ArrayList<>(); for(String key : parts.keySet()) { if(key.matches(partNameMatcher)) { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 16f4a50d69f9120d565f61d028b060d7776689fc..8fb1fa71138645cf91c84e20332876fe2093089e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -287,7 +287,7 @@ public static boolean isNonNativeTable(Table table) { } return pvals; } - public static String makePartNameMatcher(Table table, List partVals) throws MetaException { + public static String makePartNameMatcher(Table table, List partVals, String defaultStr) throws MetaException { List partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); if (partVals.size() > numPartKeys) { @@ -300,10 +300,10 @@ public static String makePartNameMatcher(Table table, List partVals) thr // or a regex of the form ".*" // This works because the "=" and "/" separating key names and partition key/values // are not escaped. - String partNameMatcher = Warehouse.makePartName(partCols, partVals, ".*"); + String partNameMatcher = Warehouse.makePartName(partCols, partVals, defaultStr); // add ".*" to the regex to match anything else afterwards the partial spec. if (partVals.size() < numPartKeys) { - partNameMatcher += ".*"; + partNameMatcher += defaultStr; } return partNameMatcher; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 08614749b7aba54f9eb9b54ac46f79dbac6bc5cd..8cd46e3f44e7c4e47fbf7f2ce2b6350a5814106f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -4698,24 +4698,10 @@ public GetPartitionsResponse get_partitions_with_specs(GetPartitionsRequest requ GetPartitionsResponse response = null; Exception ex = null; try { - List fieldList = null; - String paramkeyPattern = null; - String excludeParamKeyPattern = null; - if (request.isSetProjectionSpec()) { - GetPartitionsProjectionSpec partitionsProjectSpec = request.getProjectionSpec(); - fieldList = partitionsProjectSpec.getFieldList(); - if (partitionsProjectSpec.isSetIncludeParamKeyPattern()) { - paramkeyPattern = partitionsProjectSpec.getIncludeParamKeyPattern(); - } - if (partitionsProjectSpec.isSetExcludeParamKeyPattern()) { - excludeParamKeyPattern = partitionsProjectSpec.getExcludeParamKeyPattern(); - } - } - String dbName = parsedDbName[DB_NAME]; - Table table = get_table_core(catName, dbName, tableName); + Table table = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName); List partitions = getMS() - .getPartitionSpecsByFilterAndProjection(catName, dbName, tableName, fieldList, paramkeyPattern, - excludeParamKeyPattern); + .getPartitionSpecsByFilterAndProjection(table, request.getProjectionSpec(), + request.getFilterSpec()); List partitionSpecs = MetaStoreServerUtils.getPartitionspecsGroupedByStorageDescriptor(table, partitions); response = new GetPartitionsResponse(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index af757932a191675bc8fb9236209a2efba9f3d335..58dc6eefcb840d4dd70af7a47811fab1b5e696d9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; @@ -563,19 +564,59 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ * @param excludeParamKeyPattern The SQL regex paterrn which is used to exclude the parameter keys. Can include _ or % * When this pattern is set, all the partition parameters where key is NOT LIKE the pattern * are returned. This is applied in conjunction with the includeParamKeyPattern if it is set. + * @param filterSpec The filterSpec from GetPartitionsRequest which includes the filter mode (BY_EXPR, BY_VALUES or BY_NAMES) + * and the list of filter strings to be used to filter the results + * @param filter SqlFilterForPushDown which is set in the canUseDirectSql method before this method is called. + * The filter is used only when the mode is BY_EXPR * @return * @throws MetaException */ - public List getPartitionSpecsUsingProjection(Table tbl, - final List partitionFields, final String includeParamKeyPattern, final String excludeParamKeyPattern) + public List getPartitionsUsingProjectionAndFilterSpec(Table tbl, + final List partitionFields, final String includeParamKeyPattern, + final String excludeParamKeyPattern, GetPartitionsFilterSpec filterSpec, SqlFilterForPushdown filter) throws MetaException { final String tblName = tbl.getTableName(); final String dbName = tbl.getDbName(); final String catName = tbl.getCatName(); - //TODO add support for filter - List partitionIds = - getPartitionIdsViaSqlFilter(catName, dbName, tblName, null, Collections.emptyList(), - Collections.emptyList(), null); + List partitionIds = null; + if (filterSpec.isSetFilterMode()) { + List filters = filterSpec.getFilters(); + if (filters == null || filters.isEmpty()) { + throw new MetaException("Invalid filter expressions in the filter spec"); + } + switch(filterSpec.getFilterMode()) { + case BY_EXPR: + partitionIds = + getPartitionIdsViaSqlFilter(catName, dbName, tblName, filter.filter, filter.params, + filter.joins, null); + break; + case BY_NAMES: + String partNamesFilter = + "" + PARTITIONS + ".\"PART_NAME\" in (" + makeParams(filterSpec.getFilters().size()) + + ")"; + partitionIds = getPartitionIdsViaSqlFilter(catName, dbName, tblName, partNamesFilter, + filterSpec.getFilters(), Collections.EMPTY_LIST, null); + break; + case BY_VALUES: + // we are going to use the SQL regex pattern in the LIKE clause below. So the default string + // is _% and not .* + String partNameMatcher = MetaStoreUtils.makePartNameMatcher(tbl, filters, "_%"); + String partNamesLikeFilter = + "" + PARTITIONS + ".\"PART_NAME\" LIKE (?)"; + partitionIds = + getPartitionIdsViaSqlFilter(catName, dbName, tblName, partNamesLikeFilter, Arrays.asList(partNameMatcher), + Collections.EMPTY_LIST, null); + break; + default: + throw new MetaException("Unsupported filter mode " + filterSpec.getFilterMode()); + } + } else { + // there is no filter mode. Fetch all the partition ids + partitionIds = + getPartitionIdsViaSqlFilter(catName, dbName, tblName, null, Collections.EMPTY_LIST, + Collections.EMPTY_LIST, null); + } + if (partitionIds.isEmpty()) { return Collections.emptyList(); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 66977d79c946f1ac57aacfbe8704d37bfbac3ea3..6792192212a3ea102f2e32996b20c9b6e8ec2c1c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3255,15 +3255,12 @@ private Collection getPartitionPsQueryResults(String catName, String dbName, Str throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tableName) + " table not found"); } - String partNameMatcher = MetaStoreUtils.makePartNameMatcher(table, part_vals); + // size is known since it contains dbName, catName, tblName and partialRegex pattern + Map params = new HashMap<>(4); + String filter = getJDOFilterStrForPartitionVals(table, part_vals, params); Query query = queryWrapper.query = pm.newQuery(MPartition.class); - StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); - queryFilter.append(" && table.database.catalogName == catName"); - queryFilter.append(" && table.tableName == tableName"); - queryFilter.append(" && partitionName.matches(partialRegex)"); - query.setFilter(queryFilter.toString()); - query.declareParameters("java.lang.String dbName, java.lang.String catName, " - + "java.lang.String tableName, java.lang.String partialRegex"); + query.setFilter(filter); + query.declareParameters(makeParameterDeclarationString(params)); if (max_parts >= 0) { // User specified a row limit, set it on the Query query.setRange(0, max_parts); @@ -3272,7 +3269,7 @@ private Collection getPartitionPsQueryResults(String catName, String dbName, Str query.setResult(resultsCol); } - return (Collection) query.executeWithArray(dbName, catName, tableName, partNameMatcher); + return (Collection) query.executeWithMap(params); } @Override @@ -3360,32 +3357,27 @@ private Collection getPartitionPsQueryResults(String catName, String dbName, Str } // This code is only executed in JDO code path, not from direct SQL code path. - private List listMPartitionsWithProjection(String catName, String dbName, String tblName, int max, - QueryWrapper queryWrapper, List fieldNames) throws MetaException { + private List listMPartitionsWithProjection(QueryWrapper queryWrapper, + List fieldNames, String jdoFilter, Map params) throws MetaException { boolean success = false; List mparts = null; try { openTransaction(); LOG.debug("Executing listMPartitionsWithProjection"); - dbName = normalizeIdentifier(dbName); - tblName = normalizeIdentifier(tblName); - Query query = queryWrapper.query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + Query query = queryWrapper.query = pm.newQuery(MPartition.class, jdoFilter); + String parameterDeclaration = makeParameterDeclarationStringObj(params); + query.declareParameters(parameterDeclaration); query.setOrdering("partitionName ascending"); - if (max >= 0) { - query.setRange(0, max); - } if (fieldNames == null || fieldNames.isEmpty()) { // full fetch of partitions - mparts = (List) query.execute(tblName, dbName, catName); + mparts = (List) query.executeWithMap(params); pm.retrieveAll(mparts); } else { // fetch partially filled partitions using result clause query.setResult(Joiner.on(',').join(fieldNames)); // if more than one fields are in the result class the return type is List if (fieldNames.size() > 1) { - List results = (List) query.execute(tblName, dbName, catName); + List results = (List) query.executeWithMap(params); mparts = new ArrayList<>(results.size()); for (Object[] row : results) { MPartition mpart = new MPartition(); @@ -3398,7 +3390,7 @@ private Collection getPartitionPsQueryResults(String catName, String dbName, Str } } else { // only one field is requested, return type is List - List results = (List) query.execute(tblName, dbName, catName); + List results = (List) query.executeWithMap(params); mparts = new ArrayList<>(results.size()); for (Object row : results) { MPartition mpart = new MPartition(); @@ -3581,7 +3573,6 @@ private Integer getNumPartitionsViaOrmFilter(Table table, ExpressionTree tree, b return result.intValue(); } - /** * Gets partition names from the table via ORM (JDOQL) name filter. * @param dbName Database name. @@ -3648,12 +3639,14 @@ private void dropPartitionsNoTxn(String catName, String dbName, String tblName, return candidateCds; } - private ObjectPair> getPartQueryWithParams( - String catName, String dbName, String tblName, List partNames) { + private String getJDOFilterStrForPartitionNames(String catName, String dbName, String tblName, + List partNames, Map params) { StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); + params.put("t1", normalizeIdentifier(tblName)); + params.put("t2", normalizeIdentifier(dbName)); + params.put("t3", normalizeIdentifier(catName)); int n = 0; - Map params = new HashMap<>(); for (Iterator itr = partNames.iterator(); itr.hasNext();) { String pn = "p" + n; n++; @@ -3664,12 +3657,30 @@ private void dropPartitionsNoTxn(String catName, String dbName, String tblName, } sb.setLength(sb.length() - 4); // remove the last " || " sb.append(')'); + return sb.toString(); + } + + private String getJDOFilterStrForPartitionVals(Table table, List vals, + Map params) throws MetaException { + String partNameMatcher = MetaStoreUtils.makePartNameMatcher(table, vals, ".*"); + StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); + queryFilter.append(" && table.database.catalogName == catName"); + queryFilter.append(" && table.tableName == tableName"); + queryFilter.append(" && partitionName.matches(partialRegex)"); + params.put("dbName", table.getDbName()); + params.put("catName", table.getCatName()); + params.put("tableName", table.getTableName()); + params.put("partialRegex", partNameMatcher); + return queryFilter.toString(); + } + + private ObjectPair> getPartQueryWithParams( + String catName, String dbName, String tblName, List partNames) { Query query = pm.newQuery(); - query.setFilter(sb.toString()); - LOG.debug(" JDOQL filter is {}", sb); - params.put("t1", normalizeIdentifier(tblName)); - params.put("t2", normalizeIdentifier(dbName)); - params.put("t3", normalizeIdentifier(catName)); + Map params = new HashMap<>(); + String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); + query.setFilter(filterStr); + LOG.debug(" JDOQL filter is {}", filterStr); query.declareParameters(makeParameterDeclarationString(params)); return new ObjectPair<>(query, params); } @@ -4040,37 +4051,112 @@ protected boolean canUseDirectSql(GetHelper> ctx) throws MetaExc } @Override - public List getPartitionSpecsByFilterAndProjection(String catName, String dbName, - String tblName, List fieldList, - String includeParamKeyPattern, - String excludeParamKeyPattern) - throws MetaException, NoSuchObjectException { + public List getPartitionSpecsByFilterAndProjection(final Table table, + GetPartitionsProjectionSpec partitionsProjectSpec, + final GetPartitionsFilterSpec filterSpec) throws MetaException, NoSuchObjectException { + List fieldList = null; + String inputIncludePattern = null; + String inputExcludePattern = null; + if (partitionsProjectSpec != null) { + fieldList = partitionsProjectSpec.getFieldList(); + if (partitionsProjectSpec.isSetIncludeParamKeyPattern()) { + inputIncludePattern = partitionsProjectSpec.getIncludeParamKeyPattern(); + } + if (partitionsProjectSpec.isSetExcludeParamKeyPattern()) { + inputExcludePattern = partitionsProjectSpec.getExcludeParamKeyPattern(); + } + } if (fieldList == null || fieldList.isEmpty()) { // no fields are requested. Fallback to regular getPartitions implementation to return all the fields - return getPartitionsInternal(catName, dbName, tblName, -1, true, true); + return getPartitionsInternal(table.getCatName(), table.getDbName(), table.getTableName(), -1, + true, true); } - return new GetListHelper(catName, dbName, tblName, + // anonymous class below requires final String objects + final String includeParamKeyPattern = inputIncludePattern; + final String excludeParamKeyPattern = inputExcludePattern; + + return new GetListHelper(table.getCatName(), table.getDbName(), table.getTableName(), fieldList, true, true) { + private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); + private ExpressionTree tree; + + @Override + protected boolean canUseDirectSql(GetHelper> ctx) throws MetaException { + if (filterSpec.isSetFilterMode() && filterSpec.getFilterMode().equals(PartitionFilterMode.BY_EXPR)) { + // if the filter mode is BY_EXPR initialize the filter and generate the expression tree + // if there are more than one filter string we AND them together + initExpressionTree(); + return directSql.generateSqlFilterForPushdown(ctx.getTable(), tree, filter); + } + // BY_VALUES and BY_NAMES are always supported + return true; + } + + private void initExpressionTree() throws MetaException { + StringBuilder filterBuilder = new StringBuilder(); + int len = filterSpec.getFilters().size(); + List filters = filterSpec.getFilters(); + for (int i = 0; i < len; i++) { + filterBuilder.append('('); + filterBuilder.append(filters.get(i)); + filterBuilder.append(')'); + if (i + 1 < len) { + filterBuilder.append(" AND "); + } + } + String filterStr = filterBuilder.toString(); + tree = PartFilterExprUtil.getFilterParser(filterStr).tree; + } @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { return directSql - .getPartitionSpecsUsingProjection(ctx.getTable(), ctx.partitionFields, includeParamKeyPattern, - excludeParamKeyPattern); + .getPartitionsUsingProjectionAndFilterSpec(ctx.getTable(), ctx.partitionFields, + includeParamKeyPattern, excludeParamKeyPattern, filterSpec, filter); } @Override protected List getJdoResult( GetHelper> ctx) throws MetaException { - // For single-valued fields we can use setResult() to implement projection of fields but - // JDO doesn't support multi-valued fields in setResult() so currently JDO implementation - // fallbacks to full-partition fetch if the requested fields contain multi-valued fields - // TODO: Add param filtering logic List fieldNames = PartitionProjectionEvaluator.getMPartitionFieldNames(ctx.partitionFields); + Map params = new HashMap<>(); + String jdoFilter = null; + if (filterSpec.isSetFilterMode()) { + // generate the JDO filter string + switch(filterSpec.getFilterMode()) { + case BY_EXPR: + if (tree == null) { + // tree could be null when directSQL is disabled + initExpressionTree(); + } + jdoFilter = + makeQueryFilterString(table.getCatName(), table.getDbName(), table, tree, params, + true); + if (jdoFilter == null) { + throw new MetaException("Could not generate JDO filter from given expression"); + } + break; + case BY_NAMES: + jdoFilter = getJDOFilterStrForPartitionNames(table.getCatName(), table.getDbName(), + table.getTableName(), filterSpec.getFilters(), params); + break; + case BY_VALUES: + jdoFilter = getJDOFilterStrForPartitionVals(table, filterSpec.getFilters(), params); + break; + default: + throw new MetaException("Unsupported filter mode " + filterSpec.getFilterMode()); + } + } else { + // filter mode is not set create simple JDOFilterStr and params + jdoFilter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"; + params.put("t1", normalizeIdentifier(tblName)); + params.put("t2", normalizeIdentifier(dbName)); + params.put("t3", normalizeIdentifier(catName)); + } try (QueryWrapper queryWrapper = new QueryWrapper()) { return convertToParts( - listMPartitionsWithProjection(catName, dbName, tblName, -1, queryWrapper, fieldNames)); + listMPartitionsWithProjection(queryWrapper, fieldNames, jdoFilter, params)); } } }.run(true); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index a6d9583364be20758444ebe25c8cf636f0ea740f..c3914b668fac18ead6196a4fc449e909f5af01b1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -554,35 +554,34 @@ Partition alterPartition(String catName, String db_name, String tbl_name, Listsd.location, - * serdeInfo.name, sd.cols.name, sd.cols.type will - * return partitions which will have location field set in the storage descriptor. - * Also the serdeInf in the returned storage descriptor will only have name field - * set. This applies to multi-valued fields as well like sd.cols, so in the - * example above only name and type fields will be set for sd.cols. - * If the fieldList is empty or not present, all the fields will be set. - * @param includeParamKeyPattern SQL-92 compliant regex pattern for param keys to be included - * _ or % wildcards are supported. '_' represent one character and - * '%' represents 0 or more characters. - * @param excludeParamKeyPattern SQL-92 compliant regex pattern for param keys to be excluded - * _ or % wildcards are supported. '_' represent one character and - * '%' represents 0 or more characters + * @param table table for which whose partitions are requested + * * @param table table for which partitions are requested + * @param projectionSpec the projection spec from the GetPartitionsRequest + * This projection spec includes a fieldList which represents the fields which must be returned. + * Any other field which is not in the fieldList may be unset in the returned + * partitions (it is up to the implementation to decide whether it chooses to + * include or exclude such fields). E.g. setting the field list to sd.location, + * serdeInfo.name, sd.cols.name, sd.cols.type will + * return partitions which will have location field set in the storage descriptor. + * Also the serdeInf in the returned storage descriptor will only have name field + * set. This applies to multi-valued fields as well like sd.cols, so in the + * example above only name and type fields will be set for sd.cols. + * If the fieldList is empty or not present, all the fields will be set. + * Additionally, it also includes a includeParamKeyPattern and excludeParamKeyPattern + * which is a SQL-92 compliant regex pattern to include or exclude parameters. The paramKeyPattern + * supports _ or % wildcards which represent one character and 0 or more characters respectively + * @param filterSpec The filter spec from GetPartitionsRequest which includes the filter mode + * and the list of filter strings. The filter mode could be BY_NAMES, BY_VALUES or BY_EXPR + * to filter by partition names, partition values or expression. The filter strings are provided + * in the list of filters within the filterSpec. When more than one filters are provided in the list + * they are logically AND together * @return List of matching partitions which which may be partially filled according to fieldList. - * @throws MetaException in case of errors - * @throws NoSuchObjectException when catalog or database or table isn't found + * @throws MetaException in case of errors + * @throws NoSuchObjectException when table isn't found */ - List getPartitionSpecsByFilterAndProjection(String catName, String dbName, String tblName, - final List fieldList, - final String includeParamKeyPattern, - final String excludeParamKeyPattern) + List getPartitionSpecsByFilterAndProjection(Table table, + GetPartitionsProjectionSpec projectionSpec, GetPartitionsFilterSpec filterSpec) throws MetaException, NoSuchObjectException; - /** * Get partitions using an already parsed expression. * @param catName catalog name. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 70490f09e765d4e42391c67eb5cf018e93ad04aa..47ac68c667bea8f09f5301a6364c854bc18b3c0d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1274,13 +1274,10 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, /** * getPartitionSpecsByFilterAndProjection interface is currently non-cacheable. */ - public List getPartitionSpecsByFilterAndProjection(String catName, String dbName, - String tblName, - List fieldList, - String includeParamKeyPattern, - String excludeParamKeysPattern) throws MetaException, NoSuchObjectException { - return rawStore.getPartitionSpecsByFilterAndProjection(catName, dbName, tblName, fieldList, - includeParamKeyPattern, excludeParamKeysPattern); + public List getPartitionSpecsByFilterAndProjection(Table table, + GetPartitionsProjectionSpec projectionSpec, GetPartitionsFilterSpec filterSpec) + throws MetaException, NoSuchObjectException { + return rawStore.getPartitionSpecsByFilterAndProjection(table, projectionSpec, filterSpec); } @Override diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 4dd4edccd66f8ea8ea189a2d27f970c8113e3a0f..966979891b71f1cbfe50f56c40c35af8b304c47f 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -18,6 +18,21 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; +import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; +import org.apache.hadoop.hive.metastore.api.ISchemaName; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.TableName; @@ -376,11 +391,10 @@ public Partition alterPartition(String catName, String dbName, String tblName, L } @Override - public List getPartitionSpecsByFilterAndProjection(String catName, String dbName, String tblName, - List fieldList, String paramKeys, String excludeFlag) + public List getPartitionSpecsByFilterAndProjection(Table table, + GetPartitionsProjectionSpec projectionSpec, GetPartitionsFilterSpec filterSpec) throws MetaException, NoSuchObjectException { - return objectStore.getPartitionSpecsByFilterAndProjection(catName, - dbName, tblName, fieldList, paramKeys, excludeFlag); + return objectStore.getPartitionSpecsByFilterAndProjection(table, projectionSpec, filterSpec); } @Override diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 06f4cbce58c16f98257e7f529ffe31c983f2919f..593d562c3498660861201f58d83c27d59d184046 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; +import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.Catalog; @@ -380,8 +382,8 @@ public Partition alterPartition(String catName, String db_name, String tbl_name, } @Override - public List getPartitionSpecsByFilterAndProjection(String catName, String dbName, String tblName, - List fieldList, String paramKeys, String excludeFlag) + public List getPartitionSpecsByFilterAndProjection(Table table, + GetPartitionsProjectionSpec projectSpec, GetPartitionsFilterSpec filterSpec) throws MetaException, NoSuchObjectException { return Collections.emptyList(); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjection.java deleted file mode 100644 index dcff606937157e63694657b42392875d50b17be6..0000000000000000000000000000000000000000 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjection.java +++ /dev/null @@ -1,700 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; -import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; -import org.apache.hadoop.hive.metastore.api.GetPartitionsRequest; -import org.apache.hadoop.hive.metastore.api.GetPartitionsResponse; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec; -import org.apache.hadoop.hive.metastore.api.PartitionSpec; -import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD; -import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; -import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; -import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.thrift.TException; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import static org.apache.hadoop.hive.metastore.ColumnType.SERIALIZATION_FORMAT; - -/** - * Tests for getPartitionsWithSpecs metastore API. This test create some partitions and makes sure - * that getPartitionsWithSpecs returns results which are comparable with the get_partitions API when - * various combinations of projection spec are set. Also checks the JDO code path in addition to - * directSQL code path - */ -@Category(MetastoreCheckinTest.class) -public class TestGetPartitionsUsingProjection { - private static final Logger LOG = LoggerFactory.getLogger(TestGetPartitionsUsingProjection.class); - protected static Configuration conf = MetastoreConf.newMetastoreConf(); - private static int port; - private static final String dbName = "test_projection_db"; - private static final String tblName = "test_projection_table"; - private List origPartitions; - private Table tbl; - private static final String EXCLUDE_KEY_PREFIX = "exclude"; - private HiveMetaStoreClient client; - - @BeforeClass - public static void startMetaStoreServer() throws Exception { - conf.set("hive.in.test", "true"); - MetaStoreTestUtils.setConfForStandloneMode(conf); - MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2); - MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, 100); - port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); - LOG.info("Starting MetaStore Server on port " + port); - - try (HiveMetaStoreClient client = createClient()) { - new DatabaseBuilder().setName(dbName).create(client, conf); - } - } - - @AfterClass - public static void tearDown() throws Exception { - try (HiveMetaStoreClient client = createClient()) { - client.dropDatabase(dbName, true, true, true); - } - } - - @Before - public void setup() throws TException { - // This is default case with setugi off for both client and server - client = createClient(); - createTestTables(); - origPartitions = client.listPartitions(dbName, tblName, (short) -1); - tbl = client.getTable(dbName, tblName); - // set directSQL to true explicitly - client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "true"); - client.setMetaConf(ConfVars.TRY_DIRECT_SQL_DDL.getVarname(), "true"); - } - - @After - public void cleanup() { - dropTestTables(); - client.close(); - client = null; - } - - private void dropTestTables() { - try { - client.dropTable(dbName, tblName); - } catch (TException e) { - // ignored - } - } - - private void createTestTables() throws TException { - if (client.tableExists(dbName, tblName)) { - LOG.info("Table is already existing. Dropping it and then recreating"); - client.dropTable(dbName, tblName); - } - new TableBuilder().setTableName(tblName).setDbName(dbName).setCols(Arrays - .asList(new FieldSchema("col1", "string", "c1 comment"), - new FieldSchema("col2", "int", "c2 comment"))).setPartCols(Arrays - .asList(new FieldSchema("state", "string", "state comment"), - new FieldSchema("city", "string", "city comment"))) - .setTableParams(new HashMap(2) {{ - put("tableparam1", "tableval1"); - put("tableparam2", "tableval2"); - }}) - .setBucketCols(Collections.singletonList("col1")) - .addSortCol("col2", 1) - .addSerdeParam(SERIALIZATION_FORMAT, "1").setSerdeName(tblName) - .setSerdeLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") - .setInputFormat("org.apache.hadoop.hive.ql.io.HiveInputFormat") - .setOutputFormat("org.apache.hadoop.hive.ql.io.HiveOutputFormat") - .create(client, conf); - - Table table = client.getTable(dbName, tblName); - Assert.assertTrue("Table " + dbName + "." + tblName + " does not exist", - client.tableExists(dbName, tblName)); - - List partitions = new ArrayList<>(); - partitions.add(createPartition(Arrays.asList("CA", "SanFrancisco"), table)); - partitions.add(createPartition(Arrays.asList("CA", "PaloAlto"), table)); - partitions.add(createPartition(Arrays.asList("WA", "Seattle"), table)); - partitions.add(createPartition(Arrays.asList("AZ", "Phoenix"), table)); - - client.add_partitions(partitions); - } - - private Partition createPartition(List vals, Table table) throws MetaException { - return new PartitionBuilder() - .inTable(table) - .setValues(vals) - .addPartParam("key1", "S1") - .addPartParam("key2", "S2") - .addPartParam(EXCLUDE_KEY_PREFIX + "key1", "e1") - .addPartParam(EXCLUDE_KEY_PREFIX + "key2", "e2") - .setBucketCols(table.getSd().getBucketCols()) - .setSortCols(table.getSd().getSortCols()) - .setSerdeName(table.getSd().getSerdeInfo().getName()) - .setSerdeLib(table.getSd().getSerdeInfo().getSerializationLib()) - .setSerdeParams(table.getSd().getSerdeInfo().getParameters()) - .build(conf); - } - - private static HiveMetaStoreClient createClient() throws MetaException { - MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); - MetastoreConf.setBoolVar(conf, ConfVars.EXECUTE_SET_UGI, false); - return new HiveMetaStoreClient(conf); - } - - @Test - public void testGetPartitions() throws TException { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - validateBasic(response); - } - - @Test - public void testPartitionProjectionEmptySpec() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - - projectSpec.setFieldList(new ArrayList<>(0)); - projectSpec.setExcludeParamKeyPattern("exclude%"); - - GetPartitionsResponse response; - response = client.getPartitionsWithSpecs(request); - Assert.assertEquals(1, response.getPartitionSpec().size()); - PartitionSpec partitionSpec = response.getPartitionSpec().get(0); - PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); - - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull(sharedSD); - // everything except location in sharedSD should be same - StorageDescriptor origSd = origPartitions.get(0).getSd().deepCopy(); - origSd.unsetLocation(); - StorageDescriptor sharedSDCopy = sharedSD.deepCopy(); - sharedSDCopy.unsetLocation(); - Assert.assertEquals(origSd, sharedSDCopy); - - List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); - Assert.assertNotNull(partitionWithoutSDS); - Assert.assertEquals("Unexpected number of partitions returned", - origPartitions.size(), partitionWithoutSDS.size()); - for (int i = 0; i < origPartitions.size(); i++) { - Partition origPartition = origPartitions.get(i); - PartitionWithoutSD retPartition = partitionWithoutSDS.get(i); - Assert.assertEquals(origPartition.getCreateTime(), retPartition.getCreateTime()); - Assert.assertEquals(origPartition.getLastAccessTime(), retPartition.getLastAccessTime()); - Assert.assertEquals(origPartition.getSd().getLocation(), - sharedSD.getLocation() + retPartition.getRelativePath()); - validateMap(origPartition.getParameters(), retPartition.getParameters()); - validateList(origPartition.getValues(), retPartition.getValues()); - } - } - - @Test - public void testPartitionProjectionAllSingleValuedFields() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - - List projectedFields = Arrays - .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", - "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", - "sd.serdeInfo.name", "sd.serdeInfo.serializationLib"/*, "sd.serdeInfo.serdeType"*/); - //TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo - projectSpec.setFieldList(projectedFields); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - Assert.assertEquals(1, response.getPartitionSpec().size()); - PartitionSpec partitionSpec = response.getPartitionSpec().get(0); - Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); - Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); - PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); - - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull(sharedSD); - List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); - Assert.assertNotNull(partitionWithoutSDS); - Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); - comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); - } - - @Test - public void testProjectionUsingJDO() throws Throwable { - // disable direct SQL to make sure - client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "false"); - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - List projectedFields = Collections.singletonList("sd.location"); - projectSpec.setFieldList(projectedFields); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - Assert.assertEquals(1, response.getPartitionSpec().size()); - PartitionSpec partitionSpec = response.getPartitionSpec().get(0); - Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); - Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); - PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); - - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull(sharedSD); - List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); - Assert.assertNotNull(partitionWithoutSDS); - Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); - comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); - - // set all the single-valued fields and try using JDO - request = getGetPartitionsRequest(); - projectSpec = request.getProjectionSpec(); - projectedFields = Arrays - .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", - "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", - "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.serdeType", - "sd.serdeInfo.serializerClass", "sd.serdeInfo.deserializerClass"); - projectSpec.setFieldList(projectedFields); - - response = client.getPartitionsWithSpecs(request); - Assert.assertEquals(1, response.getPartitionSpec().size()); - partitionSpec = response.getPartitionSpec().get(0); - Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); - Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); - partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); - - sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull(sharedSD); - partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); - Assert.assertNotNull(partitionWithoutSDS); - Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); - comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); - } - - /** - * Confirms if the partitionWithoutSD object at partitionWithoutSDSIndex index has all the - * projected fields set to values which are same as the ones set in origPartitions - * @param projectedFields - * @param sharedSD - * @param partitionWithoutSDS - * @param partitionWithoutSDSIndex - * @throws IllegalAccessException - * @throws InvocationTargetException - * @throws NoSuchMethodException - */ - private void comparePartitionForSingleValuedFields(List projectedFields, - StorageDescriptor sharedSD, List partitionWithoutSDS, int partitionWithoutSDSIndex) - throws IllegalAccessException, InvocationTargetException, NoSuchMethodException { - for (Partition origPart : origPartitions) { - for (String projectField : projectedFields) { - // dbname, tableName and catName is not stored in partition - if (projectField.equals("dbName") || projectField.equals("tableName") || projectField - .equals("catName")) - continue; - if (projectField.startsWith("sd")) { - String sdPropertyName = projectField.substring(projectField.indexOf("sd.") + 3); - if (sdPropertyName.equals("location")) { - // in case of location sharedSD has the base location and partition has relative location - Assert.assertEquals("Location does not match", origPart.getSd().getLocation(), - sharedSD.getLocation() + partitionWithoutSDS.get(partitionWithoutSDSIndex).getRelativePath()); - } else { - Assert.assertEquals(PropertyUtils.getNestedProperty(origPart, projectField), - PropertyUtils.getNestedProperty(sharedSD, sdPropertyName)); - } - } else { - Assert.assertEquals(PropertyUtils.getNestedProperty(origPart, projectField), - PropertyUtils.getNestedProperty(partitionWithoutSDS.get(partitionWithoutSDSIndex), projectField)); - } - } - partitionWithoutSDSIndex++; - } - } - - @Test - public void testPartitionProjectionAllMultiValuedFields() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - List projectedFields = Arrays - .asList("values", "parameters", "sd.cols", "sd.bucketCols", "sd.sortCols", "sd.parameters", - "sd.skewedInfo", "sd.serdeInfo.parameters"); - projectSpec.setFieldList(projectedFields); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - Assert.assertEquals(1, response.getPartitionSpec().size()); - PartitionSpec partitionSpec = response.getPartitionSpec().get(0); - PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); - Assert.assertEquals(origPartitions.size(), partitionSpecWithSharedSD.getPartitions().size()); - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - for (int i = 0; i < origPartitions.size(); i++) { - Partition origPartition = origPartitions.get(i); - PartitionWithoutSD retPartition = partitionSpecWithSharedSD.getPartitions().get(i); - for (String projectedField : projectedFields) { - switch (projectedField) { - case "values": - validateList(origPartition.getValues(), retPartition.getValues()); - break; - case "parameters": - validateMap(origPartition.getParameters(), retPartition.getParameters()); - break; - case "sd.cols": - validateList(origPartition.getSd().getCols(), sharedSD.getCols()); - break; - case "sd.bucketCols": - validateList(origPartition.getSd().getBucketCols(), sharedSD.getBucketCols()); - break; - case "sd.sortCols": - validateList(origPartition.getSd().getSortCols(), sharedSD.getSortCols()); - break; - case "sd.parameters": - validateMap(origPartition.getSd().getParameters(), sharedSD.getParameters()); - break; - case "sd.skewedInfo": - if (!origPartition.getSd().getSkewedInfo().getSkewedColNames().isEmpty()) { - validateList(origPartition.getSd().getSkewedInfo().getSkewedColNames(), - sharedSD.getSkewedInfo().getSkewedColNames()); - } - if (!origPartition.getSd().getSkewedInfo().getSkewedColValues().isEmpty()) { - for (int i1 = 0; - i1 < origPartition.getSd().getSkewedInfo().getSkewedColValuesSize(); i1++) { - validateList(origPartition.getSd().getSkewedInfo().getSkewedColValues().get(i1), - sharedSD.getSkewedInfo().getSkewedColValues().get(i1)); - } - } - if (!origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps().isEmpty()) { - validateMap(origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(), - sharedSD.getSkewedInfo().getSkewedColValueLocationMaps()); - } - break; - case "sd.serdeInfo.parameters": - validateMap(origPartition.getSd().getSerdeInfo().getParameters(), - sharedSD.getSerdeInfo().getParameters()); - break; - default: - throw new IllegalArgumentException("Invalid field " + projectedField); - } - } - } - } - - @Test - public void testPartitionProjectionIncludeParameters() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - projectSpec - .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); - projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - Assert.assertNotNull("All the partitions should be returned in sharedSD spec", - partitionSpecWithSharedSD); - PartitionListComposingSpec partitionListComposingSpec = - response.getPartitionSpec().get(0).getPartitionList(); - Assert.assertNull("Partition list composing spec should be null since all the " - + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); - for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { - Assert.assertTrue("included parameter key is not found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); - Assert.assertTrue("included parameter key is not found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); - Assert.assertEquals("Additional parameters returned other than inclusion keys", - 2, retPartion.getParameters().size()); - } - } - - @Test - public void testPartitionProjectionIncludeExcludeParameters() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - projectSpec - .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); - // test parameter key inclusion using setIncludeParamKeyPattern - projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); - projectSpec.setExcludeParamKeyPattern("%key1%"); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - Assert.assertNotNull("All the partitions should be returned in sharedSD spec", - partitionSpecWithSharedSD); - PartitionListComposingSpec partitionListComposingSpec = - response.getPartitionSpec().get(0).getPartitionList(); - Assert.assertNull("Partition list composing spec should be null since all the " - + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); - for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { - Assert.assertFalse("excluded parameter key is found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); - Assert.assertTrue("included parameter key is not found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); - Assert.assertEquals("Additional parameters returned other than inclusion keys", - 1, retPartion.getParameters().size()); - } - } - - @Test - public void testPartitionProjectionExcludeParameters() throws Throwable { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - projectSpec - .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); - projectSpec.setExcludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - Assert.assertNotNull("All the partitions should be returned in sharedSD spec", - partitionSpecWithSharedSD); - PartitionListComposingSpec partitionListComposingSpec = - response.getPartitionSpec().get(0).getPartitionList(); - Assert.assertNull("Partition list composing spec should be null", partitionListComposingSpec); - for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { - Assert.assertFalse("excluded parameter key is found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); - Assert.assertFalse("excluded parameter key is found in the response", - retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); - } - } - - @Test - public void testNestedMultiValuedFieldProjection() throws TException { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - projectSpec.setFieldList(Arrays.asList("sd.cols.name", "sd.cols.type")); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); - for (FieldSchema col : sharedSD.getCols()) { - Assert.assertTrue("sd.cols.name was requested but was not returned", col.isSetName()); - Assert.assertTrue("sd.cols.type was requested but was not returned", col.isSetType()); - Assert.assertFalse("sd.cols.comment was not requested but was returned", col.isSetComment()); - } - } - - @Test - public void testParameterExpansion() throws TException { - GetPartitionsRequest request = getGetPartitionsRequest(); - GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); - projectSpec.setFieldList(Arrays.asList("sd.cols", "sd.serdeInfo")); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); - Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); - Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", - origPartitions.get(0).getSd().getCols(), sharedSD.getCols()); - - Assert - .assertNotNull("sd.serdeInfo were requested but was not returned", sharedSD.getSerdeInfo()); - Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", - origPartitions.get(0).getSd().getSerdeInfo(), sharedSD.getSerdeInfo()); - } - - @Test - public void testNonStandardPartitions() throws TException { - String testTblName = "test_non_standard"; - new TableBuilder() - .setTableName(testTblName) - .setDbName(dbName) - .addCol("ns_c1", "string", "comment 1") - .addCol("ns_c2", "int", "comment 2") - .addPartCol("part", "string") - .addPartCol("city", "string") - .addBucketCol("ns_c1") - .addSortCol("ns_c2", 1) - .addTableParam("tblparamKey", "Partitions of this table are not located within table directory") - .create(client, conf); - - Table table = client.getTable(dbName, testTblName); - Assert.assertNotNull("Unable to create a test table ", table); - - List partitions = new ArrayList<>(); - partitions.add(createPartition(Arrays.asList("p1", "SanFrancisco"), table)); - partitions.add(createPartition(Arrays.asList("p1", "PaloAlto"), table)); - partitions.add(createPartition(Arrays.asList("p2", "Seattle"), table)); - partitions.add(createPartition(Arrays.asList("p2", "Phoenix"), table)); - - client.add_partitions(partitions); - // change locations of two of the partitions outside table directory - List testPartitions = client.listPartitions(dbName, testTblName, (short) -1); - Assert.assertEquals(4, testPartitions.size()); - Partition p1 = testPartitions.get(2); - p1.getSd().setLocation("/tmp/some_other_location/part=p2/city=Seattle"); - Partition p2 = testPartitions.get(3); - p2.getSd().setLocation("/tmp/some_other_location/part=p2/city=Phoenix"); - client.alter_partitions(dbName, testTblName, Arrays.asList(p1, p2)); - - GetPartitionsRequest request = getGetPartitionsRequest(); - request.getProjectionSpec().setFieldList(Arrays.asList("values", "sd")); - request.setDbName(dbName); - request.setTblName(testTblName); - - GetPartitionsResponse response = client.getPartitionsWithSpecs(request); - Assert.assertNotNull("Response should have returned partition specs", - response.getPartitionSpec()); - Assert - .assertEquals("We should have two partition specs", 2, response.getPartitionSpec().size()); - Assert.assertNotNull("One SharedSD spec is expected", - response.getPartitionSpec().get(0).getSharedSDPartitionSpec()); - Assert.assertNotNull("One composing spec is expected", - response.getPartitionSpec().get(1).getPartitionList()); - - PartitionSpecWithSharedSD partitionSpecWithSharedSD = - response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); - Assert.assertNotNull("sd was requested but not returned", partitionSpecWithSharedSD.getSd()); - Assert.assertEquals("shared SD should have table location", table.getSd().getLocation(), - partitionSpecWithSharedSD.getSd().getLocation()); - List> expectedVals = new ArrayList<>(2); - expectedVals.add(Arrays.asList("p1", "PaloAlto")); - expectedVals.add(Arrays.asList("p1", "SanFrancisco")); - - for (int i=0; i partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); - Assert.assertEquals(origPartitions.size(), partitionWithoutSDS.size()); - for (int i = 0; i < origPartitions.size(); i++) { - Partition origPartition = origPartitions.get(i); - PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i); - Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), - origPartition.getSd().getLocation(), - sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath()); - } - } - - private GetPartitionsRequest getGetPartitionsRequest() { - GetPartitionsRequest request = new GetPartitionsRequest(); - request.setProjectionSpec(new GetPartitionsProjectionSpec()); - request.setFilterSpec(new GetPartitionsFilterSpec()); - request.setTblName(tblName); - request.setDbName(dbName); - return request; - } - - private void validateMap(Map aMap, Map bMap) { - if ((aMap == null || aMap.isEmpty()) && (bMap == null || bMap.isEmpty())) { - return; - } - // Equality is verified here because metastore updates stats automatically - // and adds them in the returned partition. So the returned partition will - // have parameters + some more parameters for the basic stats - Assert.assertTrue(bMap.size() >= aMap.size()); - for (Entry entries : aMap.entrySet()) { - Assert.assertTrue("Expected " + entries.getKey() + " is missing from the map", - bMap.containsKey(entries.getKey())); - Assert.assertEquals("Expected value to be " + aMap.get(entries.getKey()) + " found" + bMap - .get(entries.getKey()), aMap.get(entries.getKey()), bMap.get(entries.getKey())); - } - } - - private void validateList(List aList, List bList) { - if ((aList == null || aList.isEmpty()) && (bList == null || bList.isEmpty())) { - return; - } - Assert.assertEquals(aList.size(), bList.size()); - Iterator origValuesIt = aList.iterator(); - Iterator retValuesIt = bList.iterator(); - while (origValuesIt.hasNext()) { - Assert.assertTrue(retValuesIt.hasNext()); - Assert.assertEquals(origValuesIt.next(), retValuesIt.next()); - } - } -} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java new file mode 100644 index 0000000000000000000000000000000000000000..bc43f3d551b7d4338247b72d64c85fad5eb3f323 --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java @@ -0,0 +1,904 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.commons.beanutils.PropertyUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; +import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec; +import org.apache.hadoop.hive.metastore.api.GetPartitionsRequest; +import org.apache.hadoop.hive.metastore.api.GetPartitionsResponse; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionFilterMode; +import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD; +import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import static org.apache.hadoop.hive.metastore.ColumnType.SERIALIZATION_FORMAT; + +/** + * Tests for getPartitionsWithSpecs metastore API. This test create some partitions and makes sure + * that getPartitionsWithSpecs returns results which are comparable with the get_partitions API when + * various combinations of projection spec are set. Also checks the JDO code path in addition to + * directSQL code path + */ +@Category(MetastoreCheckinTest.class) +public class TestGetPartitionsUsingProjectionAndFilterSpecs { + private static final Logger LOG = LoggerFactory.getLogger(TestGetPartitionsUsingProjectionAndFilterSpecs.class); + protected static Configuration conf = MetastoreConf.newMetastoreConf(); + private static int port; + private static final String dbName = "test_projection_db"; + private static final String tblName = "test_projection_table"; + private List origPartitions; + private Table tbl; + private static final String EXCLUDE_KEY_PREFIX = "exclude"; + private HiveMetaStoreClient client; + + @BeforeClass + public static void startMetaStoreServer() throws Exception { + conf.set("hive.in.test", "true"); + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2); + MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, 100); + port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + LOG.info("Starting MetaStore Server on port " + port); + + try (HiveMetaStoreClient client = createClient()) { + new DatabaseBuilder().setName(dbName).create(client, conf); + } + } + + @AfterClass + public static void tearDown() throws Exception { + try (HiveMetaStoreClient client = createClient()) { + client.dropDatabase(dbName, true, true, true); + } + } + + @Before + public void setup() throws TException { + // This is default case with setugi off for both client and server + client = createClient(); + createTestTables(); + origPartitions = client.listPartitions(dbName, tblName, (short) -1); + tbl = client.getTable(dbName, tblName); + // set directSQL to true explicitly + client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "true"); + client.setMetaConf(ConfVars.TRY_DIRECT_SQL_DDL.getVarname(), "true"); + } + + @After + public void cleanup() { + dropTestTables(); + client.close(); + client = null; + } + + private void dropTestTables() { + try { + client.dropTable(dbName, tblName); + } catch (TException e) { + // ignored + } + } + + private void createTestTables() throws TException { + if (client.tableExists(dbName, tblName)) { + LOG.info("Table is already existing. Dropping it and then recreating"); + client.dropTable(dbName, tblName); + } + new TableBuilder().setTableName(tblName).setDbName(dbName).setCols(Arrays + .asList(new FieldSchema("col1", "string", "c1 comment"), + new FieldSchema("col2", "int", "c2 comment"))).setPartCols(Arrays + .asList(new FieldSchema("state", "string", "state comment"), + new FieldSchema("city", "string", "city comment"))) + .setTableParams(new HashMap(2) {{ + put("tableparam1", "tableval1"); + put("tableparam2", "tableval2"); + }}) + .setBucketCols(Collections.singletonList("col1")) + .addSortCol("col2", 1) + .addSerdeParam(SERIALIZATION_FORMAT, "1").setSerdeName(tblName) + .setSerdeLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") + .setInputFormat("org.apache.hadoop.hive.ql.io.HiveInputFormat") + .setOutputFormat("org.apache.hadoop.hive.ql.io.HiveOutputFormat") + .create(client, conf); + + Table table = client.getTable(dbName, tblName); + Assert.assertTrue("Table " + dbName + "." + tblName + " does not exist", + client.tableExists(dbName, tblName)); + + List partitions = new ArrayList<>(); + partitions.add(createPartition(Arrays.asList("CA", "SanFrancisco"), table)); + partitions.add(createPartition(Arrays.asList("CA", "PaloAlto"), table)); + partitions.add(createPartition(Arrays.asList("WA", "Seattle"), table)); + partitions.add(createPartition(Arrays.asList("AZ", "Phoenix"), table)); + + client.add_partitions(partitions); + } + + private Partition createPartition(List vals, Table table) throws MetaException { + return new PartitionBuilder() + .inTable(table) + .setValues(vals) + .addPartParam("key1", "S1") + .addPartParam("key2", "S2") + .addPartParam(EXCLUDE_KEY_PREFIX + "key1", "e1") + .addPartParam(EXCLUDE_KEY_PREFIX + "key2", "e2") + .setBucketCols(table.getSd().getBucketCols()) + .setSortCols(table.getSd().getSortCols()) + .setSerdeName(table.getSd().getSerdeInfo().getName()) + .setSerdeLib(table.getSd().getSerdeInfo().getSerializationLib()) + .setSerdeParams(table.getSd().getSerdeInfo().getParameters()) + .build(conf); + } + + private static HiveMetaStoreClient createClient() throws MetaException { + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + MetastoreConf.setBoolVar(conf, ConfVars.EXECUTE_SET_UGI, false); + return new HiveMetaStoreClient(conf); + } + + @Test + public void testGetPartitions() throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + validateBasic(response); + } + + @Test + public void testPartitionProjectionEmptySpec() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + + projectSpec.setFieldList(new ArrayList<>(0)); + projectSpec.setExcludeParamKeyPattern("exclude%"); + + GetPartitionsResponse response; + response = client.getPartitionsWithSpecs(request); + Assert.assertEquals(1, response.getPartitionSpec().size()); + PartitionSpec partitionSpec = response.getPartitionSpec().get(0); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); + + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull(sharedSD); + // everything except location in sharedSD should be same + StorageDescriptor origSd = origPartitions.get(0).getSd().deepCopy(); + origSd.unsetLocation(); + StorageDescriptor sharedSDCopy = sharedSD.deepCopy(); + sharedSDCopy.unsetLocation(); + Assert.assertEquals(origSd, sharedSDCopy); + + List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); + Assert.assertNotNull(partitionWithoutSDS); + Assert.assertEquals("Unexpected number of partitions returned", + origPartitions.size(), partitionWithoutSDS.size()); + for (int i = 0; i < origPartitions.size(); i++) { + Partition origPartition = origPartitions.get(i); + PartitionWithoutSD retPartition = partitionWithoutSDS.get(i); + Assert.assertEquals(origPartition.getCreateTime(), retPartition.getCreateTime()); + Assert.assertEquals(origPartition.getLastAccessTime(), retPartition.getLastAccessTime()); + Assert.assertEquals(origPartition.getSd().getLocation(), + sharedSD.getLocation() + retPartition.getRelativePath()); + validateMap(origPartition.getParameters(), retPartition.getParameters()); + validateList(origPartition.getValues(), retPartition.getValues()); + } + } + + @Test + public void testPartitionProjectionAllSingleValuedFields() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + + List projectedFields = Arrays + .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", + "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", + "sd.serdeInfo.name", "sd.serdeInfo.serializationLib"/*, "sd.serdeInfo.serdeType"*/); + //TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo + projectSpec.setFieldList(projectedFields); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertEquals(1, response.getPartitionSpec().size()); + PartitionSpec partitionSpec = response.getPartitionSpec().get(0); + Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); + Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); + + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull(sharedSD); + List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); + Assert.assertNotNull(partitionWithoutSDS); + Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); + comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); + } + + @Test + public void testProjectionUsingJDO() throws Throwable { + // disable direct SQL to make sure + client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "false"); + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + List projectedFields = Collections.singletonList("sd.location"); + projectSpec.setFieldList(projectedFields); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertEquals(1, response.getPartitionSpec().size()); + PartitionSpec partitionSpec = response.getPartitionSpec().get(0); + Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); + Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); + + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull(sharedSD); + List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); + Assert.assertNotNull(partitionWithoutSDS); + Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); + comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); + + // set all the single-valued fields and try using JDO + request = getGetPartitionsRequest(); + projectSpec = request.getProjectionSpec(); + projectedFields = Arrays + .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", + "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", + "sd.serdeInfo.name", "sd.serdeInfo.serializationLib", "sd.serdeInfo.serdeType", + "sd.serdeInfo.serializerClass", "sd.serdeInfo.deserializerClass"); + projectSpec.setFieldList(projectedFields); + + response = client.getPartitionsWithSpecs(request); + Assert.assertEquals(1, response.getPartitionSpec().size()); + partitionSpec = response.getPartitionSpec().get(0); + Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); + Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); + partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); + + sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull(sharedSD); + partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); + Assert.assertNotNull(partitionWithoutSDS); + Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); + comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); + } + + /** + * Confirms if the partitionWithoutSD object at partitionWithoutSDSIndex index has all the + * projected fields set to values which are same as the ones set in origPartitions + * @param projectedFields + * @param sharedSD + * @param partitionWithoutSDS + * @param partitionWithoutSDSIndex + * @throws IllegalAccessException + * @throws InvocationTargetException + * @throws NoSuchMethodException + */ + private void comparePartitionForSingleValuedFields(List projectedFields, + StorageDescriptor sharedSD, List partitionWithoutSDS, int partitionWithoutSDSIndex) + throws IllegalAccessException, InvocationTargetException, NoSuchMethodException { + for (Partition origPart : origPartitions) { + for (String projectField : projectedFields) { + // dbname, tableName and catName is not stored in partition + if (projectField.equals("dbName") || projectField.equals("tableName") || projectField + .equals("catName")) + continue; + if (projectField.startsWith("sd")) { + String sdPropertyName = projectField.substring(projectField.indexOf("sd.") + 3); + if (sdPropertyName.equals("location")) { + // in case of location sharedSD has the base location and partition has relative location + Assert.assertEquals("Location does not match", origPart.getSd().getLocation(), + sharedSD.getLocation() + partitionWithoutSDS.get(partitionWithoutSDSIndex).getRelativePath()); + } else { + Assert.assertEquals(PropertyUtils.getNestedProperty(origPart, projectField), + PropertyUtils.getNestedProperty(sharedSD, sdPropertyName)); + } + } else { + Assert.assertEquals(PropertyUtils.getNestedProperty(origPart, projectField), + PropertyUtils.getNestedProperty(partitionWithoutSDS.get(partitionWithoutSDSIndex), projectField)); + } + } + partitionWithoutSDSIndex++; + } + } + + @Test + public void testPartitionProjectionAllMultiValuedFields() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + List projectedFields = Arrays + .asList("values", "parameters", "sd.cols", "sd.bucketCols", "sd.sortCols", "sd.parameters", + "sd.skewedInfo", "sd.serdeInfo.parameters"); + projectSpec.setFieldList(projectedFields); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + Assert.assertEquals(1, response.getPartitionSpec().size()); + PartitionSpec partitionSpec = response.getPartitionSpec().get(0); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); + Assert.assertEquals(origPartitions.size(), partitionSpecWithSharedSD.getPartitions().size()); + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + for (int i = 0; i < origPartitions.size(); i++) { + Partition origPartition = origPartitions.get(i); + PartitionWithoutSD retPartition = partitionSpecWithSharedSD.getPartitions().get(i); + for (String projectedField : projectedFields) { + switch (projectedField) { + case "values": + validateList(origPartition.getValues(), retPartition.getValues()); + break; + case "parameters": + validateMap(origPartition.getParameters(), retPartition.getParameters()); + break; + case "sd.cols": + validateList(origPartition.getSd().getCols(), sharedSD.getCols()); + break; + case "sd.bucketCols": + validateList(origPartition.getSd().getBucketCols(), sharedSD.getBucketCols()); + break; + case "sd.sortCols": + validateList(origPartition.getSd().getSortCols(), sharedSD.getSortCols()); + break; + case "sd.parameters": + validateMap(origPartition.getSd().getParameters(), sharedSD.getParameters()); + break; + case "sd.skewedInfo": + if (!origPartition.getSd().getSkewedInfo().getSkewedColNames().isEmpty()) { + validateList(origPartition.getSd().getSkewedInfo().getSkewedColNames(), + sharedSD.getSkewedInfo().getSkewedColNames()); + } + if (!origPartition.getSd().getSkewedInfo().getSkewedColValues().isEmpty()) { + for (int i1 = 0; + i1 < origPartition.getSd().getSkewedInfo().getSkewedColValuesSize(); i1++) { + validateList(origPartition.getSd().getSkewedInfo().getSkewedColValues().get(i1), + sharedSD.getSkewedInfo().getSkewedColValues().get(i1)); + } + } + if (!origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps().isEmpty()) { + validateMap(origPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(), + sharedSD.getSkewedInfo().getSkewedColValueLocationMaps()); + } + break; + case "sd.serdeInfo.parameters": + validateMap(origPartition.getSd().getSerdeInfo().getParameters(), + sharedSD.getSerdeInfo().getParameters()); + break; + default: + throw new IllegalArgumentException("Invalid field " + projectedField); + } + } + } + } + + @Test + public void testPartitionProjectionIncludeParameters() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec + .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); + projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull("All the partitions should be returned in sharedSD spec", + partitionSpecWithSharedSD); + PartitionListComposingSpec partitionListComposingSpec = + response.getPartitionSpec().get(0).getPartitionList(); + Assert.assertNull("Partition list composing spec should be null since all the " + + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); + for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { + Assert.assertTrue("included parameter key is not found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); + Assert.assertTrue("included parameter key is not found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); + Assert.assertEquals("Additional parameters returned other than inclusion keys", + 2, retPartion.getParameters().size()); + } + } + + @Test + public void testPartitionProjectionIncludeExcludeParameters() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec + .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); + // test parameter key inclusion using setIncludeParamKeyPattern + projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); + projectSpec.setExcludeParamKeyPattern("%key1%"); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull("All the partitions should be returned in sharedSD spec", + partitionSpecWithSharedSD); + PartitionListComposingSpec partitionListComposingSpec = + response.getPartitionSpec().get(0).getPartitionList(); + Assert.assertNull("Partition list composing spec should be null since all the " + + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); + for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { + Assert.assertFalse("excluded parameter key is found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); + Assert.assertTrue("included parameter key is not found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); + Assert.assertEquals("Additional parameters returned other than inclusion keys", + 1, retPartion.getParameters().size()); + } + } + + @Test + public void testPartitionProjectionExcludeParameters() throws Throwable { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec + .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); + projectSpec.setExcludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull("All the partitions should be returned in sharedSD spec", + partitionSpecWithSharedSD); + PartitionListComposingSpec partitionListComposingSpec = + response.getPartitionSpec().get(0).getPartitionList(); + Assert.assertNull("Partition list composing spec should be null", partitionListComposingSpec); + for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { + Assert.assertFalse("excluded parameter key is found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); + Assert.assertFalse("excluded parameter key is found in the response", + retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); + } + } + + @Test + public void testNestedMultiValuedFieldProjection() throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(Arrays.asList("sd.cols.name", "sd.cols.type")); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); + for (FieldSchema col : sharedSD.getCols()) { + Assert.assertTrue("sd.cols.name was requested but was not returned", col.isSetName()); + Assert.assertTrue("sd.cols.type was requested but was not returned", col.isSetType()); + Assert.assertFalse("sd.cols.comment was not requested but was returned", col.isSetComment()); + } + } + + @Test + public void testParameterExpansion() throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(Arrays.asList("sd.cols", "sd.serdeInfo")); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); + Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", + origPartitions.get(0).getSd().getCols(), sharedSD.getCols()); + + Assert + .assertNotNull("sd.serdeInfo were requested but was not returned", sharedSD.getSerdeInfo()); + Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", + origPartitions.get(0).getSd().getSerdeInfo(), sharedSD.getSerdeInfo()); + } + + @Test + public void testNonStandardPartitions() throws TException { + String testTblName = "test_non_standard"; + new TableBuilder() + .setTableName(testTblName) + .setDbName(dbName) + .addCol("ns_c1", "string", "comment 1") + .addCol("ns_c2", "int", "comment 2") + .addPartCol("part", "string") + .addPartCol("city", "string") + .addBucketCol("ns_c1") + .addSortCol("ns_c2", 1) + .addTableParam("tblparamKey", "Partitions of this table are not located within table directory") + .create(client, conf); + + Table table = client.getTable(dbName, testTblName); + Assert.assertNotNull("Unable to create a test table ", table); + + List partitions = new ArrayList<>(); + partitions.add(createPartition(Arrays.asList("p1", "SanFrancisco"), table)); + partitions.add(createPartition(Arrays.asList("p1", "PaloAlto"), table)); + partitions.add(createPartition(Arrays.asList("p2", "Seattle"), table)); + partitions.add(createPartition(Arrays.asList("p2", "Phoenix"), table)); + + client.add_partitions(partitions); + // change locations of two of the partitions outside table directory + List testPartitions = client.listPartitions(dbName, testTblName, (short) -1); + Assert.assertEquals(4, testPartitions.size()); + Partition p1 = testPartitions.get(2); + p1.getSd().setLocation("/tmp/some_other_location/part=p2/city=Seattle"); + Partition p2 = testPartitions.get(3); + p2.getSd().setLocation("/tmp/some_other_location/part=p2/city=Phoenix"); + client.alter_partitions(dbName, testTblName, Arrays.asList(p1, p2)); + + GetPartitionsRequest request = getGetPartitionsRequest(); + request.getProjectionSpec().setFieldList(Arrays.asList("values", "sd")); + request.setDbName(dbName); + request.setTblName(testTblName); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertNotNull("Response should have returned partition specs", + response.getPartitionSpec()); + Assert + .assertEquals("We should have two partition specs", 2, response.getPartitionSpec().size()); + Assert.assertNotNull("One SharedSD spec is expected", + response.getPartitionSpec().get(0).getSharedSDPartitionSpec()); + Assert.assertNotNull("One composing spec is expected", + response.getPartitionSpec().get(1).getPartitionList()); + + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull("sd was requested but not returned", partitionSpecWithSharedSD.getSd()); + Assert.assertEquals("shared SD should have table location", table.getSd().getLocation(), + partitionSpecWithSharedSD.getSd().getLocation()); + List> expectedVals = new ArrayList<>(2); + expectedVals.add(Arrays.asList("p1", "PaloAlto")); + expectedVals.add(Arrays.asList("p1", "SanFrancisco")); + + for (int i=0; i values) throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(values); + client.getPartitionsWithSpecs(request); + } + + private void runGetPartitionsUsingExpr() throws TException { + // test simple case first + getPartitionsWithExpr(Arrays.asList("state=\"CA\""), 2); + // Logical AND in filter + getPartitionsWithExpr(Arrays.asList("state=\"CA\" AND city=\"PaloAlto\""), 1); + // empty result set + getPartitionsWithExpr(Arrays.asList("state=\"CA\" AND city=\"Seattle\""), 0); + + // NOT operator + getPartitionsWithExpr(Arrays.asList("state=\"CA\" AND city !=\"PaloAlto\""), 1); + // nested expr + getPartitionsWithExpr(Arrays + .asList("(state=\"CA\" AND city !=\"PaloAlto\") OR (state=\"WA\" AND city = \"Seattle\")"), + 2); + + // multiple filters + getPartitionsWithExpr(Arrays.asList("state=\"CA\"", "city=\"PaloAlto\""), 1); + getPartitionsWithExpr( + Arrays.asList("state=\"CA\" OR state=\"WA\"", "city=\"PaloAlto\" OR city=\"Seattle\""), 2); + // test empty result + getPartitionsWithExpr(Arrays.asList("state=\"AZ\"", "city=\"Tucson\""), 0); + } + + private void getPartitionsWithExpr(List filters, int expectedPartition) throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(Arrays.asList("sd.location")); + request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_EXPR); + request.getFilterSpec().setFilters(filters); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertNotNull(response); + if (expectedPartition > 0) { + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull(partitionSpecWithSharedSD); + Assert.assertEquals("Invalid number of partitions returned", expectedPartition, + partitionSpecWithSharedSD.getPartitionsSize()); + } else { + Assert.assertTrue( + "Partition spec should have been empty since filter doesn't match with any partitions", + response.getPartitionSpec().isEmpty()); + } + } + + private void getPartitionsWithVals(List filters, int expectedPartitions) + throws TException { + // get partitions from "trusted" API + List partitions = client.listPartitions(dbName, tblName, filters, (short) -1); + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(Arrays.asList("sd.location")); + request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_VALUES); + request.getFilterSpec().setFilters(filters); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertNotNull(response); + if (expectedPartitions > 0) { + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull(partitionSpecWithSharedSD); + Assert.assertEquals("Invalid number of partitions returned", expectedPartitions, + partitionSpecWithSharedSD.getPartitionsSize()); + verifyLocations(partitions, partitionSpecWithSharedSD.getSd(), + partitionSpecWithSharedSD.getPartitions()); + } else { + Assert.assertTrue( + "Partition spec should have been empty since filter doesn't match with any partitions", + response.getPartitionSpec().isEmpty()); + } + } + + private void runGetPartitionsUsingVals() throws TException { + // top level val set + getPartitionsWithVals(Arrays.asList("CA"), 2); + // exactly one partition + getPartitionsWithVals(Arrays.asList("CA", "PaloAlto"), 1); + // non-existing partition should return zero partitions + getPartitionsWithVals(Arrays.asList("CA", "CityDoesNotExist"), 0); + } + + private void getPartitionsWithNames(List names, int expectedPartitionCount) throws TException { + GetPartitionsRequest request = getGetPartitionsRequest(); + GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); + projectSpec.setFieldList(Arrays.asList("sd.location")); + request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_NAMES); + request.getFilterSpec().setFilters(names); + + GetPartitionsResponse response = client.getPartitionsWithSpecs(request); + Assert.assertNotNull(response); + if (expectedPartitionCount > 0) { + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull(partitionSpecWithSharedSD); + Assert.assertEquals("Invalid number of partitions returned", expectedPartitionCount, + partitionSpecWithSharedSD.getPartitionsSize()); + List origPartitions = client.getPartitionsByNames(dbName, tblName, names); + verifyLocations(origPartitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions()); + } else { + Assert.assertTrue( + "Partition spec should have been empty since filter doesn't match with any partitions", + response.getPartitionSpec().isEmpty()); + } + } + + private void runGetPartitionsUsingNames() throws TException { + List names = client.listPartitionNames(dbName, tblName, (short) -1); + // remove one to make sure that the test is really looking at 3 names + names.remove(names.size() - 1); + getPartitionsWithNames(names, 3); + // filter mode is set. Empty filter names. So no partitions should be returned + getPartitionsWithNames(Arrays.asList(""), 0); + // invalid name + getPartitionsWithNames(Arrays.asList("invalidPartitionName"), 0); + } + + private void validateBasic(GetPartitionsResponse response) throws TException { + Assert.assertNotNull("Response is null", response); + Assert.assertNotNull("Returned partition spec is null", response.getPartitionSpec()); + Assert.assertEquals(1, response.getPartitionSpecSize()); + PartitionSpecWithSharedSD partitionSpecWithSharedSD = + response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); + Assert.assertNotNull(partitionSpecWithSharedSD.getSd()); + StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); + Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), + sharedSD.getLocation()); + + List partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); + Assert.assertEquals(origPartitions.size(), partitionWithoutSDS.size()); + for (int i = 0; i < origPartitions.size(); i++) { + Partition origPartition = origPartitions.get(i); + PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i); + Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), + origPartition.getSd().getLocation(), + sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath()); + } + } + + private GetPartitionsRequest getGetPartitionsRequest() { + GetPartitionsRequest request = new GetPartitionsRequest(); + request.setProjectionSpec(new GetPartitionsProjectionSpec()); + request.setFilterSpec(new GetPartitionsFilterSpec()); + request.setTblName(tblName); + request.setDbName(dbName); + return request; + } + + private void verifyLocations(List origPartitions, StorageDescriptor sharedSD, + List partitionWithoutSDS) { + int i=0; + for (Partition origPart : origPartitions) { + // in case of location sharedSD has the base location and partition has relative location + Assert.assertEquals("Location does not match", origPart.getSd().getLocation(), + sharedSD.getLocation() + partitionWithoutSDS.get(i).getRelativePath()); + Assert.assertNull("values were not requested but are still set", + partitionWithoutSDS.get(i).getValues()); + Assert.assertNull("Parameters were not requested but are still set", + partitionWithoutSDS.get(i).getParameters()); + i++; + } + } + + private void validateMap(Map aMap, Map bMap) { + if ((aMap == null || aMap.isEmpty()) && (bMap == null || bMap.isEmpty())) { + return; + } + // Equality is verified here because metastore updates stats automatically + // and adds them in the returned partition. So the returned partition will + // have parameters + some more parameters for the basic stats + Assert.assertTrue(bMap.size() >= aMap.size()); + for (Entry entries : aMap.entrySet()) { + Assert.assertTrue("Expected " + entries.getKey() + " is missing from the map", + bMap.containsKey(entries.getKey())); + Assert.assertEquals("Expected value to be " + aMap.get(entries.getKey()) + " found" + bMap + .get(entries.getKey()), aMap.get(entries.getKey()), bMap.get(entries.getKey())); + } + } + + private void validateList(List aList, List bList) { + if ((aList == null || aList.isEmpty()) && (bList == null || bList.isEmpty())) { + return; + } + Assert.assertEquals(aList.size(), bList.size()); + Iterator origValuesIt = aList.iterator(); + Iterator retValuesIt = bList.iterator(); + while (origValuesIt.hasNext()) { + Assert.assertTrue(retValuesIt.hasNext()); + Assert.assertEquals(origValuesIt.next(), retValuesIt.next()); + } + } +}