commit bf7dd674e2b29553efa7c2dfa7c92d8cca0e1a89 Author: Owen O'Malley Date: Fri Aug 21 17:04:16 2015 -0700 HIVE-11618. Correct the SARG api to unify int and double. diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 0d765b1..fcb3746 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -637,8 +637,6 @@ private static Object getBaseObjectForComparison(PredicateLeaf.Type type, Object return ((BigDecimal) obj).doubleValue(); } break; - case INTEGER: - // fall through case LONG: if (obj instanceof Number) { // widening conversion diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java index a1dbc1a..1ceea6e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/LeafFilterFactory.java @@ -22,6 +22,8 @@ import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; import static org.apache.parquet.filter2.predicate.FilterApi.eq; import static org.apache.parquet.filter2.predicate.FilterApi.lt; @@ -146,12 +148,16 @@ public FilterPredicate buildPredict(Operator op, Object constant, * @param type FilterPredicateType * @return */ - public FilterPredicateLeafBuilder getLeafFilterBuilderByType(PredicateLeaf.Type type){ + public FilterPredicateLeafBuilder getLeafFilterBuilderByType(PredicateLeaf.Type type, + Type parquetType){ switch (type){ - case INTEGER: - return new IntFilterPredicateLeafBuilder(); case LONG: - return new LongFilterPredicateLeafBuilder(); + if (parquetType.asPrimitiveType().getPrimitiveTypeName() == + PrimitiveType.PrimitiveTypeName.INT32) { + return new IntFilterPredicateLeafBuilder(); + } else { + return new LongFilterPredicateLeafBuilder(); + } case FLOAT: // float and double return new DoubleFilterPredicateLeafBuilder(); case STRING: // string, char, varchar diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java index f170026..d1864ae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java @@ -37,14 +37,6 @@ private static final Log LOG = LogFactory.getLog(ParquetFilterPredicateConverter.class); /** - * Translate the search argument to the filter predicate parquet uses - * @return translate the sarg into a filter predicate - */ - public static FilterPredicate toFilterPredicate(SearchArgument sarg) { - return toFilterPredicate(sarg, null); - } - - /** * Translate the search argument to the filter predicate parquet uses. It includes * only the columns from the passed schema. * @return translate the sarg into a filter predicate @@ -58,18 +50,21 @@ public static FilterPredicate toFilterPredicate(SearchArgument sarg, MessageType } } - return translate(sarg.getExpression(), sarg.getLeaves(), columns); + return translate(sarg.getExpression(), sarg.getLeaves(), columns, schema); } - private static FilterPredicate translate(ExpressionTree root, List leaves, Set columns) { + private static FilterPredicate translate(ExpressionTree root, + List leaves, + Set columns, + MessageType schema) { FilterPredicate p = null; switch (root.getOperator()) { case OR: for(ExpressionTree child: root.getChildren()) { if (p == null) { - p = translate(child, leaves, columns); + p = translate(child, leaves, columns, schema); } else { - FilterPredicate right = translate(child, leaves, columns); + FilterPredicate right = translate(child, leaves, columns, schema); // constant means no filter, ignore it when it is null if(right != null){ p = FilterApi.or(p, right); @@ -80,9 +75,9 @@ private static FilterPredicate translate(ExpressionTree root, List args = new ArrayList(); - args.add(10); - args.add(20); + args.add(10L); + args.add(20L); PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf - (PredicateLeaf.Operator.IN, PredicateLeaf.Type.INTEGER, + (PredicateLeaf.Operator.IN, PredicateLeaf.Type.LONG, "x", null, args); assertEquals(TruthValue.YES_NULL, RecordReaderImpl.evaluatePredicateProto(createIntStats(20L, 20L), pred, null)); @@ -727,10 +727,10 @@ public void testIn() throws Exception { @Test public void testBetween() throws Exception { List args = new ArrayList(); - args.add(10); - args.add(20); + args.add(10L); + args.add(20L); PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf - (PredicateLeaf.Operator.BETWEEN, PredicateLeaf.Type.INTEGER, + (PredicateLeaf.Operator.BETWEEN, PredicateLeaf.Type.LONG, "x", null, args); assertEquals(TruthValue.NO_NULL, RecordReaderImpl.evaluatePredicateProto(createIntStats(0L, 5L), pred, null)); @@ -751,7 +751,7 @@ public void testBetween() throws Exception { @Test public void testIsNull() throws Exception { PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf - (PredicateLeaf.Operator.IS_NULL, PredicateLeaf.Type.INTEGER, + (PredicateLeaf.Operator.IS_NULL, PredicateLeaf.Type.LONG, "x", null, null); assertEquals(TruthValue.YES_NO, RecordReaderImpl.evaluatePredicateProto(createIntStats(20L, 30L), pred, null)); @@ -1306,10 +1306,10 @@ public void testIntEqualsBloomFilter() throws Exception { @Test public void testIntInBloomFilter() throws Exception { List args = new ArrayList(); - args.add(15); - args.add(19); + args.add(15L); + args.add(19L); PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf - (PredicateLeaf.Operator.IN, PredicateLeaf.Type.INTEGER, + (PredicateLeaf.Operator.IN, PredicateLeaf.Type.LONG, "x", null, args); BloomFilterIO bf = new BloomFilterIO(10000); for (int i = 20; i < 1000; i++) { diff --git ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRecordReaderWrapper.java ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRecordReaderWrapper.java index f9ca528..e92b696 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRecordReaderWrapper.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestParquetRecordReaderWrapper.java @@ -28,6 +28,8 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.junit.Test; import java.sql.Date; @@ -48,15 +50,19 @@ public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) - .between("y", PredicateLeaf.Type.INTEGER, 10, 20) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .isNull("x", PredicateLeaf.Type.LONG) + .between("y", PredicateLeaf.Type.LONG, 10L, 20L) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = MessageTypeParser.parseMessageType("message test {" + + " optional int32 x; required int32 y; required int32 z;" + + " optional binary a;}"); + FilterPredicate p = + ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(and(not(eq(x, null)), not(and(lt(y, 20), not(lteq(y, 10))))), not(or(or(eq(z, 1), " + "eq(z, 2)), eq(z, 3)))), not(eq(a, Binary{\"stinger\"})))"; @@ -75,23 +81,27 @@ public void testBuilderComplexTypes() throws Exception { .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); + MessageType schema = MessageTypeParser.parseMessageType("message test {" + + " required int32 x; required binary y; required binary z;}"); assertEquals("lteq(y, Binary{\"hi \"})", - ParquetFilterPredicateConverter.toFilterPredicate(sarg).toString()); + ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema).toString()); sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) + .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); - - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + schema = MessageTypeParser.parseMessageType("message test {" + + " optional int32 x; required binary y; required int32 z;" + + " optional binary a;}"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(not(eq(x, null)), not(or(or(eq(z, 1), eq(z, 2)), eq(z, 3)))), " + "not(eq(a, Binary{\"stinger\"})))"; @@ -110,23 +120,28 @@ public void testBuilderComplexTypes2() throws Exception { new HiveDecimalWritable("1.0")) .end() .build(); + MessageType schema = MessageTypeParser.parseMessageType("message test {" + + " required int32 x; required binary y; required binary z;}"); assertEquals("lteq(y, Binary{\"hi \"})", - ParquetFilterPredicateConverter.toFilterPredicate(sarg).toString()); + ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema).toString()); sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) + .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); + schema = MessageTypeParser.parseMessageType("message test {" + + " optional int32 x; required binary y; required int32 z;" + + " optional binary a;}"); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(not(eq(x, null)), not(or(or(eq(z, 1), eq(z, 2)), eq(z, 3)))), " + "not(eq(a, Binary{\"stinger\"})))"; assertEquals(expected, p.toString()); @@ -137,16 +152,19 @@ public void testBuilderFloat() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() - .lessThan("x", PredicateLeaf.Type.INTEGER, new Integer((short) 22)) - .lessThan("x1", PredicateLeaf.Type.INTEGER, new Integer(22)) + .lessThan("x", PredicateLeaf.Type.LONG, 22L) + .lessThan("x1", PredicateLeaf.Type.LONG, 22L) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.FLOAT, new Double(0.22)) .equals("z1", PredicateLeaf.Type.FLOAT, new Double(0.22)) .end() .build(); + MessageType schema = MessageTypeParser.parseMessageType("message test {" + + " required int32 x; required int32 x1;" + + " required binary y; required float z; required float z1;}"); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(and(and(lt(x, 22), lt(x1, 22))," + " lteq(y, Binary{\"hi \"})), eq(z, " + "0.22)), eq(z1, 0.22))"; diff --git ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java index 847a02b..ac5c1a0 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java @@ -35,9 +35,9 @@ public void testFilterColumnsThatDoNoExistOnSchema() { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("a", PredicateLeaf.Type.INTEGER) - .between("y", PredicateLeaf.Type.INTEGER, 10, 20) // Column will be removed from filter - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) // Column will be removed from filter + .isNull("a", PredicateLeaf.Type.LONG) + .between("y", PredicateLeaf.Type.LONG, 10L, 20L) // Column will be removed from filter + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) // Column will be removed from filter .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() diff --git ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestConvertAstToSearchArg.java ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestConvertAstToSearchArg.java index 9e8425a..e72789d 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestConvertAstToSearchArg.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestConvertAstToSearchArg.java @@ -28,6 +28,8 @@ import org.apache.hadoop.hive.ql.io.parquet.read.ParquetFilterPredicateConverter; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.junit.Test; import java.beans.XMLDecoder; @@ -550,7 +552,11 @@ public void testExpression1() throws Exception { List leaves = sarg.getLeaves(); assertEquals(9, leaves.size()); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String[] conditions = new String[]{ "eq(first_name, Binary{\"john\"})", /* first_name = 'john' */ "not(lteq(first_name, Binary{\"greg\"}))", /* 'greg' < first_name */ @@ -586,34 +592,34 @@ public void testExpression1() throws Exception { assertEquals("alan", leaf.getLiteral()); leaf = leaves.get(3); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); + assertEquals(12L, leaf.getLiteral()); leaf = leaves.get(4); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(13, leaf.getLiteral()); + assertEquals(13L, leaf.getLiteral()); leaf = leaves.get(5); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(15, leaf.getLiteral()); + assertEquals(15L, leaf.getLiteral()); leaf = leaves.get(6); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(16, leaf.getLiteral()); + assertEquals(16L, leaf.getLiteral()); leaf = leaves.get(7); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(30, leaf.getLiteral()); + assertEquals(30L, leaf.getLiteral()); leaf = leaves.get(8); assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); @@ -842,7 +848,10 @@ public void testExpression2() throws Exception { "lteq(id, 4)" /* id <= 4 */ }; - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = String.format("or(or(or(%1$s, %2$s), %3$s), %4$s)", conditions); assertEquals(expected, p.toString()); @@ -860,16 +869,16 @@ public void testExpression2() throws Exception { assertEquals("sue", leaf.getLiteral()); leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); + assertEquals(12L, leaf.getLiteral()); leaf = leaves.get(3); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(4, leaf.getLiteral()); + assertEquals(4L, leaf.getLiteral()); assertEquals("(or leaf-0 (not leaf-1) (not leaf-2) leaf-3)", sarg.getExpression().toString()); @@ -1271,18 +1280,21 @@ public void testExpression3() throws Exception { "eq(first_name, Binary{\"alan\"})", /* first_name = 'alan' */ "eq(last_name, Binary{\"smith\"})" /* 'smith' = last_name */ }; + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; required binary last_name;}"); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = String.format("and(and(and(%1$s, %2$s), %3$s), %4$s)", conditions); assertEquals(expected, p.toString()); PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.BETWEEN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); assertEquals(null, leaf.getLiteral()); - assertEquals(23, leaf.getLiteralList().get(0)); - assertEquals(45, leaf.getLiteralList().get(1)); + assertEquals(23L, leaf.getLiteralList().get(0)); + assertEquals(45L, leaf.getLiteralList().get(1)); leaf = leaves.get(1); assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); @@ -1493,15 +1505,19 @@ id in (34,50) */ "or(eq(id, 34), eq(id, 50))" /* id in (34,50) */ }; - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = + ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = String.format("and(and(%1$s, %2$s), %3$s)", conditions); assertEquals(expected, p.toString()); PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); + assertEquals(12L, leaf.getLiteral()); leaf = leaves.get(1); assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); @@ -1511,11 +1527,11 @@ id in (34,50) */ assertEquals("sue", leaf.getLiteralList().get(1)); leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.IN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(34, leaf.getLiteralList().get(0)); - assertEquals(50, leaf.getLiteralList().get(1)); + assertEquals(34L, leaf.getLiteralList().get(0)); + assertEquals(50L, leaf.getLiteralList().get(1)); assertEquals("(and (not leaf-0) leaf-1 leaf-2)", sarg.getExpression().toString()); @@ -1752,7 +1768,10 @@ public void testExpression5() throws Exception { List leaves = sarg.getLeaves(); assertEquals(1, leaves.size()); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(lt(first_name, Binary{\"greg\"}), not(lteq(first_name, Binary{\"david\"})))"; assertEquals(p.toString(), expected); @@ -2232,7 +2251,10 @@ public void testExpression7() throws Exception { List leaves = sarg.getLeaves(); assertEquals(9, leaves.size()); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(and(" + "or(or(or(lt(id, 18), lt(id, 10)), lt(id, 13)), lt(id, 16)), " + "or(or(or(lt(id, 18), lt(id, 11)), lt(id, 13)), lt(id, 16))), " + @@ -2255,58 +2277,58 @@ public void testExpression7() throws Exception { assertEquals(p.toString(), expected); PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(18, leaf.getLiteral()); + assertEquals(18L, leaf.getLiteral()); leaf = leaves.get(1); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(10, leaf.getLiteral()); + assertEquals(10L, leaf.getLiteral()); leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(13, leaf.getLiteral()); + assertEquals(13L, leaf.getLiteral()); leaf = leaves.get(3); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(16, leaf.getLiteral()); + assertEquals(16L, leaf.getLiteral()); leaf = leaves.get(4); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(11, leaf.getLiteral()); + assertEquals(11L, leaf.getLiteral()); leaf = leaves.get(5); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); + assertEquals(12L, leaf.getLiteral()); leaf = leaves.get(6); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(14, leaf.getLiteral()); + assertEquals(14L, leaf.getLiteral()); leaf = leaves.get(7); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(15, leaf.getLiteral()); + assertEquals(15L, leaf.getLiteral()); leaf = leaves.get(8); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); + assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); assertEquals("id", leaf.getColumnName()); - assertEquals(17, leaf.getLiteral()); + assertEquals(17L, leaf.getLiteral()); assertEquals("(and" + " (or leaf-0 leaf-1 leaf-2 leaf-3)" + @@ -2388,7 +2410,10 @@ public void testExpression8() throws Exception { List leaves = sarg.getLeaves(); assertEquals(0, leaves.size()); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); assertNull(p); assertEquals("YES_NO_NULL", @@ -2643,15 +2668,18 @@ public void testExpression10() throws Exception { List leaves = sarg.getLeaves(); assertEquals(1, leaves.size()); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg); + MessageType schema = + MessageTypeParser.parseMessageType("message test { required int32 id;" + + " required binary first_name; }"); + FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(not(lt(id, 10)), not(lt(id, 10)))"; assertEquals(expected, p.toString()); - assertEquals(PredicateLeaf.Type.INTEGER, leaves.get(0).getType()); + assertEquals(PredicateLeaf.Type.LONG, leaves.get(0).getType()); assertEquals(PredicateLeaf.Operator.LESS_THAN, leaves.get(0).getOperator()); assertEquals("id", leaves.get(0).getColumnName()); - assertEquals(10, leaves.get(0).getLiteral()); + assertEquals(10L, leaves.get(0).getLiteral()); assertEquals("(and (not leaf-0) (not leaf-0))", sarg.getExpression().toString()); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index 20de846..573d5c6 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -330,7 +330,7 @@ public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() - .lessThan("x", PredicateLeaf.Type.INTEGER, 10) + .lessThan("x", PredicateLeaf.Type.LONG, 10L) .lessThanEquals("y", PredicateLeaf.Type.STRING, "hi") .equals("z", PredicateLeaf.Type.FLOAT, 1.0) .end() @@ -342,9 +342,9 @@ public void testBuilder() throws Exception { sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) - .between("y", PredicateLeaf.Type.INTEGER, 10, 20) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .isNull("x", PredicateLeaf.Type.LONG) + .between("y", PredicateLeaf.Type.LONG, 10L, 20L) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() @@ -376,10 +376,10 @@ public void testBuilderComplexTypes() throws Exception { sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) + .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() @@ -413,10 +413,10 @@ public void testBuilderComplexTypes2() throws Exception { sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() - .isNull("x", PredicateLeaf.Type.INTEGER) + .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) - .in("z", PredicateLeaf.Type.INTEGER, 1, 2, 3) + .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() @@ -435,8 +435,8 @@ public void testBuilderFloat() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() - .lessThan("x", PredicateLeaf.Type.INTEGER, new Integer((short) 22)) - .lessThan("x1", PredicateLeaf.Type.INTEGER, new Integer(22)) + .lessThan("x", PredicateLeaf.Type.LONG, 22L) + .lessThan("x1", PredicateLeaf.Type.LONG, 22L) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.FLOAT, new Double(0.22)) @@ -480,7 +480,7 @@ public void testBadLiteral() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() - .lessThan("x", PredicateLeaf.Type.INTEGER, "hi") + .lessThan("x", PredicateLeaf.Type.LONG, "hi") .end() .build(); } diff --git storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java index 3a92565..dc71db4 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java @@ -47,8 +47,7 @@ * The possible types for sargs. */ public static enum Type { - INTEGER(Integer.class), // all of the integer types except long - LONG(Long.class), + LONG(Long.class), // all of the integer types FLOAT(Double.class), // float and double STRING(String.class), // string, char, varchar DATE(Date.class),