From a221785d161cbc3d8a91515573cedbeb5fce99d6 Mon Sep 17 00:00:00 2001 From: Sam An Date: Mon, 30 Mar 2020 18:12:55 -0700 Subject: [PATCH] MsckPartitionExpressionProxy should filter partitions Change-Id: I42da124c8da8045d7dc50a3389585798bdb5d073 --- .../MsckPartitionExpressionProxy.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckPartitionExpressionProxy.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckPartitionExpressionProxy.java index d8428255590..bc659ab74c7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckPartitionExpressionProxy.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MsckPartitionExpressionProxy.java @@ -18,12 +18,18 @@ */ import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; // This is added as part of moving MSCK code from ql to standalone-metastore. There is a metastore API to drop // partitions by name but we cannot use it because msck typically will contain partition value (year=2014). We almost @@ -36,6 +42,8 @@ // should use SearchArgument (storage-api) to construct the filter expression and not depend on ql, but the usecase // for msck is pretty simple and this specific implementation should suffice. public class MsckPartitionExpressionProxy implements PartitionExpressionProxy { + private static final Logger LOG = LoggerFactory.getLogger(MsckPartitionExpressionProxy.class); + @Override public String convertExprToFilter(final byte[] exprBytes, final String defaultPartitionName) throws MetaException { return new String(exprBytes, StandardCharsets.UTF_8); @@ -44,6 +52,38 @@ public String convertExprToFilter(final byte[] exprBytes, final String defaultPa @Override public boolean filterPartitionsByExpr(List partColumns, byte[] expr, String defaultPartitionName, List partitionNames) throws MetaException { + String partExpr = new String(expr, StandardCharsets.UTF_8); + LOG.debug(StringUtils.format("Partition expr: %s", expr)); + //This is to find in partitionNames all that match expr + //reverse of the Msck.makePartExpr + Set partValueSet = new HashSet<>(); + String[] parts = partExpr.split(" AND "); + for ( String part : parts){ + String[] colAndValue = part.split("="); + String key = colAndValue[0]; + String value = colAndValue[1].replace("'", ""); + partValueSet.add(key+"="+value); + } + + List partNamesSeq = new ArrayList<>(); + for (String partition : partitionNames){ + boolean isMatch = true; + for ( String col : partValueSet){ + if (partition.indexOf(col) == -1){ + isMatch = false; + break; + } + } + if (isMatch){ + partNamesSeq.add(partition); + } + } + partitionNames.clear(); + partitionNames.addAll(partNamesSeq); + LOG.info(StringUtils.format("The returned partition list is of size: %d", partitionNames.size())); + for(String s : partitionNames){ + LOG.debug("Matched partition: %s", s); + } return false; } -- 2.23.0