diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java index a1dbc1a..88a68da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java @@ -22,8 +22,11 @@ import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import static org.apache.parquet.filter2.predicate.FilterApi.eq; +import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn; import static org.apache.parquet.filter2.predicate.FilterApi.lt; import static org.apache.parquet.filter2.predicate.FilterApi.ltEq; import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn; @@ -81,6 +84,25 @@ public FilterPredicate buildPredict(Operator op, Object constant, } } + class FloatFilterPredicateLeafBuilder extends FilterPredicateLeafBuilder { + @Override + public FilterPredicate buildPredict(Operator op, Object constant, String columnName) { + switch (op) { + case LESS_THAN: + return lt(floatColumn(columnName), ((Number) constant).floatValue()); + case IS_NULL: + case EQUALS: + case NULL_SAFE_EQUALS: + return eq(floatColumn(columnName), + (constant == null) ? null : ((Number) constant).floatValue()); + case LESS_THAN_EQUALS: + return ltEq(FilterApi.floatColumn(columnName), ((Number) constant).floatValue()); + default: + throw new RuntimeException("Unknown PredicateLeaf Operator type: " + op); + } + } + } + class DoubleFilterPredicateLeafBuilder extends FilterPredicateLeafBuilder { @Override @@ -146,21 +168,22 @@ public FilterPredicate buildPredict(Operator op, Object constant, * @param type FilterPredicateType * @return */ - public FilterPredicateLeafBuilder getLeafFilterBuilderByType(PredicateLeaf.Type type){ + public FilterPredicateLeafBuilder getLeafFilterBuilderByType(PrimitiveType.PrimitiveTypeName type){ switch (type){ - case INTEGER: + case INT32: return new IntFilterPredicateLeafBuilder(); - case LONG: + case INT64: return new LongFilterPredicateLeafBuilder(); - case FLOAT: // float and double - return new DoubleFilterPredicateLeafBuilder(); - case STRING: // string, char, varchar + case FLOAT: + return new FloatFilterPredicateLeafBuilder(); + case BINARY: + case INT96: + case FIXED_LEN_BYTE_ARRAY: return new BinaryFilterPredicateLeafBuilder(); case BOOLEAN: return new BooleanFilterPredicateLeafBuilder(); - case DATE: - case DECIMAL: - case TIMESTAMP: + case DOUBLE: + return new DoubleFilterPredicateLeafBuilder(); default: LOG.debug("Conversion to Parquet FilterPredicate not supported for " + type); return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java index f170026..e4db133 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java @@ -27,10 +27,13 @@ import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class ParquetFilterPredicateConverter { @@ -51,25 +54,29 @@ public static FilterPredicate toFilterPredicate(SearchArgument sarg) { */ public static FilterPredicate toFilterPredicate(SearchArgument sarg, MessageType schema) { Set columns = null; + Map typeMap = new HashMap<>(); if (schema != null) { columns = new HashSet(); for (Type field : schema.getFields()) { columns.add(field.getName()); + if (field.isPrimitive()) { + typeMap.put(field.getName(), field.asPrimitiveType().getPrimitiveTypeName()); + } } } - return translate(sarg.getExpression(), sarg.getLeaves(), columns); + return translate(sarg.getExpression(), sarg.getLeaves(), columns, typeMap); } - private static FilterPredicate translate(ExpressionTree root, List leaves, Set columns) { + private static FilterPredicate translate(ExpressionTree root, List leaves, Set columns, Map typeMap) { FilterPredicate p = null; switch (root.getOperator()) { case OR: for(ExpressionTree child: root.getChildren()) { if (p == null) { - p = translate(child, leaves, columns); + p = translate(child, leaves, columns, typeMap); } else { - FilterPredicate right = translate(child, leaves, columns); + FilterPredicate right = translate(child, leaves, columns, typeMap); // constant means no filter, ignore it when it is null if(right != null){ p = FilterApi.or(p, right); @@ -80,9 +87,9 @@ private static FilterPredicate translate(ExpressionTree root, List