diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index cfa78cb..bd8aeb8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.io.orc; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_ZEROCOPY; + import java.io.EOFException; import java.io.IOException; import java.math.BigInteger; @@ -30,8 +32,8 @@ import java.util.Map; import java.util.TreeMap; -import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -40,10 +42,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.HiveConf; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ORC_ZEROCOPY; -import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; @@ -60,14 +60,15 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; +import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim; +import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.hive.shims.HadoopShims.*; import com.google.common.collect.ComparisonChain; @@ -2244,25 +2245,29 @@ static TruthValue evaluatePredicate(OrcProto.ColumnStatistics index, static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, Object max) { Location loc; - - // column statistics for char/varchar columns are stored as strings, so convert char/varchar - // type predicates to string - Object predObj = predicate.getLiteral(); - Object minValue = getPrimitiveObject(predObj, min); - Object maxValue = getPrimitiveObject(predObj, max); - - switch (predicate.getOperator()) { + try { + // Predicate object and stats object can be one of the following base types + // LONG, DOUBLE, STRING, DATE, DECIMAL + // Out of these DATE is not implicitly convertible to other types and rest + // others are implicitly convertible. In cases where DATE cannot be converted + // the stats object is converted to text and comparison is performed. + // When STRINGs are converted to other base types, NumberFormat exception + // can occur in which case TruthValue.YES_NO_NULL value is returned + Object baseObj = predicate.getLiteral(); + Object minValue = getConvertedStatsObj(min, baseObj); + Object maxValue = getConvertedStatsObj(max, baseObj); + Object predObj = getBaseObjectForComparison(baseObj, minValue); + + switch (predicate.getOperator()) { case NULL_SAFE_EQUALS: - loc = compareToRange((Comparable) predicate.getLiteral(), - minValue, maxValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (loc == Location.BEFORE || loc == Location.AFTER) { return TruthValue.NO; } else { return TruthValue.YES_NO; } case EQUALS: - loc = compareToRange((Comparable) predicate.getLiteral(), - minValue, maxValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (minValue.equals(maxValue) && loc == Location.MIN) { return TruthValue.YES_NULL; } else if (loc == Location.BEFORE || loc == Location.AFTER) { @@ -2271,8 +2276,7 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, return TruthValue.YES_NO_NULL; } case LESS_THAN: - loc = compareToRange((Comparable) predicate.getLiteral(), - minValue, maxValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (loc == Location.AFTER) { return TruthValue.YES_NULL; } else if (loc == Location.BEFORE || loc == Location.MIN) { @@ -2281,8 +2285,7 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, return TruthValue.YES_NO_NULL; } case LESS_THAN_EQUALS: - loc = compareToRange((Comparable) predicate.getLiteral(), - minValue, maxValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (loc == Location.AFTER || loc == Location.MAX) { return TruthValue.YES_NULL; } else if (loc == Location.BEFORE) { @@ -2294,10 +2297,9 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, if (minValue.equals(maxValue)) { // for a single value, look through to see if that value is in the // set - for(Object arg: predicate.getLiteralList()) { - minValue = getPrimitiveObject(arg, min); - maxValue = getPrimitiveObject(arg, max); - loc = compareToRange((Comparable) arg, minValue, maxValue); + for (Object arg : predicate.getLiteralList()) { + predObj = getBaseObjectForComparison(arg, minValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (loc == Location.MIN) { return TruthValue.YES_NULL; } @@ -2305,10 +2307,9 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, return TruthValue.NO_NULL; } else { // are all of the values outside of the range? - for(Object arg: predicate.getLiteralList()) { - minValue = getPrimitiveObject(arg, min); - maxValue = getPrimitiveObject(arg, max); - loc = compareToRange((Comparable) arg, minValue, maxValue); + for (Object arg : predicate.getLiteralList()) { + predObj = getBaseObjectForComparison(arg, minValue); + loc = compareToRange((Comparable) predObj, minValue, maxValue); if (loc == Location.MIN || loc == Location.MIDDLE || loc == Location.MAX) { return TruthValue.YES_NO_NULL; @@ -2318,17 +2319,13 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, } case BETWEEN: List args = predicate.getLiteralList(); - minValue = getPrimitiveObject(args.get(0), min); - maxValue = getPrimitiveObject(args.get(0), max); + Object predObj1 = getBaseObjectForComparison(args.get(0), minValue); - loc = compareToRange((Comparable) args.get(0), minValue, maxValue); + loc = compareToRange((Comparable) predObj1, minValue, maxValue); if (loc == Location.BEFORE || loc == Location.MIN) { - Object predObj2 = args.get(1); - minValue = getPrimitiveObject(predObj2, min); - maxValue = getPrimitiveObject(predObj2, max); + Object predObj2 = getBaseObjectForComparison(args.get(1), minValue); - Location loc2 = compareToRange((Comparable) predObj2, minValue, - maxValue); + Location loc2 = compareToRange((Comparable) predObj2, minValue, maxValue); if (loc2 == Location.AFTER || loc2 == Location.MAX) { return TruthValue.YES_NULL; } else if (loc2 == Location.BEFORE) { @@ -2345,39 +2342,61 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, return TruthValue.YES_NO; default: return TruthValue.YES_NO_NULL; + } + + // in case failed conversion, return the default YES_NO_NULL truth value + } catch (NumberFormatException nfe) { + return TruthValue.YES_NO_NULL; } } - private static Object getPrimitiveObject(Object predObj, Object obj) { - if (obj instanceof DateWritable) { - DateWritable dobj = (DateWritable) obj; - if (predObj instanceof String || predObj instanceof HiveChar - || predObj instanceof HiveVarchar) { - return dobj.toString(); - } - } else if (obj instanceof HiveDecimal) { - HiveDecimal hdObj = (HiveDecimal) obj; - if (predObj instanceof Float) { - return hdObj.floatValue(); - } else if (predObj instanceof Double) { - return hdObj.doubleValue(); - } else if (predObj instanceof Short) { - return hdObj.shortValue(); - } else if (predObj instanceof Integer) { - return hdObj.intValue(); - } else if (predObj instanceof Long) { - return hdObj.longValue(); - } else if (predObj instanceof String || predObj instanceof HiveChar - || predObj instanceof HiveVarchar) { - // primitive type of char/varchar is Text (i.e trailing white spaces trimmed string) - return StringUtils.stripEnd(hdObj.toString(), null); - } - } else if (obj instanceof String || obj instanceof HiveChar || obj instanceof HiveVarchar) { - // primitive type of char/varchar is Text (i.e trailing white spaces trimmed string) - return StringUtils.stripEnd(obj.toString(), null); - } - - return obj; + private static Object getBaseObjectForComparison(Object predObj, Object statsObj) { + if (predObj != null) { + // following are implicitly convertible + if (statsObj instanceof Long) { + if (predObj instanceof Double) { + return ((Double) predObj).longValue(); + } else if (predObj instanceof HiveDecimal) { + return ((HiveDecimal) predObj).longValue(); + } else if (predObj instanceof String) { + return Long.valueOf(predObj.toString()); + } + } else if (statsObj instanceof Double) { + if (predObj instanceof Long) { + return ((Long) predObj).doubleValue(); + } else if (predObj instanceof HiveDecimal) { + return ((HiveDecimal) predObj).doubleValue(); + } else if (predObj instanceof String) { + return Double.valueOf(predObj.toString()); + } + } else if (statsObj instanceof String) { + return predObj.toString(); + } else if (statsObj instanceof HiveDecimal) { + if (predObj instanceof Long) { + return HiveDecimal.create(((Long) predObj)); + } else if (predObj instanceof Double) { + return HiveDecimal.create(predObj.toString()); + } else if (predObj instanceof String) { + return HiveDecimal.create(predObj.toString()); + } + } + } + return predObj; + } + + private static Object getConvertedStatsObj(Object statsObj, Object predObj) { + + // converting between date and other types is not implicit, so convert to + // text for comparison + if (((predObj instanceof DateWritable) && !(statsObj instanceof DateWritable)) + || ((statsObj instanceof DateWritable) && !(predObj instanceof DateWritable))) { + return StringUtils.stripEnd(statsObj.toString(), null); + } + + if (statsObj instanceof String) { + return StringUtils.stripEnd(statsObj.toString(), null); + } + return statsObj; } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java index 922b99f..fc736a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java @@ -45,11 +45,9 @@ public static enum Type { INTEGER, // all of the integer types FLOAT, // float and double - STRING, + STRING, // string, char, varchar DATE, - DECIMAL, - CHAR, - VARCHAR + DECIMAL } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java index 4f26d3f..f8d19b6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -26,6 +26,10 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -44,6 +48,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -284,9 +289,7 @@ Operator getOperator() { case LONG: return PredicateLeaf.Type.INTEGER; case CHAR: - return PredicateLeaf.Type.CHAR; case VARCHAR: - return PredicateLeaf.Type.VARCHAR; case STRING: return PredicateLeaf.Type.STRING; case FLOAT: @@ -327,9 +330,12 @@ private static Object boxLiteral(ExprNodeConstantDesc lit) { case INTEGER: return ((Number) lit.getValue()).longValue(); case STRING: - return lit.getValue().toString(); + return StringUtils.stripEnd(lit.getValue().toString(), null); case FLOAT: return ((Number) lit.getValue()).doubleValue(); + case DATE: + case DECIMAL: + return lit; default: throw new IllegalArgumentException("Unknown literal " + getType(lit)); } @@ -900,10 +906,17 @@ public Builder end() { private static Object boxLiteral(Object literal) { if (literal instanceof String || literal instanceof Long || - literal instanceof Double) { + literal instanceof Double || + literal instanceof DateWritable || + literal instanceof HiveDecimal) { return literal; - } else if (literal instanceof Integer) { - return Long.valueOf((Integer) literal); + } else if (literal instanceof HiveChar || + literal instanceof HiveVarchar) { + return StringUtils.stripEnd(literal.toString(), null); + } else if (literal instanceof Byte || + literal instanceof Short || + literal instanceof Integer) { + return Long.valueOf(literal.toString()); } else if (literal instanceof Float) { return Double.valueOf((Float) literal); } else { @@ -913,12 +926,22 @@ private static Object boxLiteral(Object literal) { } private static PredicateLeaf.Type getType(Object literal) { - if (literal instanceof Long) { + if (literal instanceof Byte || + literal instanceof Short || + literal instanceof Integer || + literal instanceof Long) { return PredicateLeaf.Type.INTEGER; - } else if (literal instanceof String) { + } else if (literal instanceof HiveChar || + literal instanceof HiveVarchar || + literal instanceof String) { return PredicateLeaf.Type.STRING; - } else if (literal instanceof Double) { + } else if (literal instanceof Float || + literal instanceof Double) { return PredicateLeaf.Type.FLOAT; + } else if (literal instanceof DateWritable) { + return PredicateLeaf.Type.DATE; + } else if (literal instanceof HiveDecimal) { + return PredicateLeaf.Type.DECIMAL; } throw new IllegalArgumentException("Unknown type for literal " + literal); } diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java index 3595e05..bc21cc4 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.junit.Test; public class TestRecordReaderImpl { @@ -130,6 +131,20 @@ public void testGetMin() throws Exception { .setIntStatistics(intStats.build()).build(); } + private static OrcProto.ColumnStatistics createIntStats(int min, int max) { + OrcProto.IntegerStatistics.Builder intStats = OrcProto.IntegerStatistics.newBuilder(); + intStats.setMinimum(min); + intStats.setMaximum(max); + return OrcProto.ColumnStatistics.newBuilder().setIntStatistics(intStats.build()).build(); + } + + private static OrcProto.ColumnStatistics createDoubleStats(double min, double max) { + OrcProto.DoubleStatistics.Builder dblStats = OrcProto.DoubleStatistics.newBuilder(); + dblStats.setMinimum(min); + dblStats.setMaximum(max); + return OrcProto.ColumnStatistics.newBuilder().setDoubleStatistics(dblStats.build()).build(); + } + private static OrcProto.ColumnStatistics createStringStats(String min, String max) { OrcProto.StringStatistics.Builder strStats = OrcProto.StringStatistics.newBuilder(); strStats.setMinimum(min); @@ -137,6 +152,13 @@ public void testGetMin() throws Exception { return OrcProto.ColumnStatistics.newBuilder().setStringStatistics(strStats.build()).build(); } + private static OrcProto.ColumnStatistics createDateStats(int min, int max) { + OrcProto.DateStatistics.Builder dateStats = OrcProto.DateStatistics.newBuilder(); + dateStats.setMinimum(min); + dateStats.setMaximum(max); + return OrcProto.ColumnStatistics.newBuilder().setDateStatistics(dateStats.build()).build(); + } + private static OrcProto.ColumnStatistics createDecimalStats(String min, String max) { OrcProto.DecimalStatistics.Builder decStats = OrcProto.DecimalStatistics.newBuilder(); decStats.setMinimum(min); @@ -166,6 +188,178 @@ public void testGetMax() throws Exception { } @Test + public void testPredEvalWithIntStats() throws Exception { + PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( + PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.INTEGER, "x", 15L, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createIntStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.FLOAT, "x", 15.0, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createIntStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "15", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createIntStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(15), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createIntStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.create(15), null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createIntStats(10, 100), pred)); + } + + @Test + public void testPredEvalWithDoubleStats() throws Exception { + PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( + PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.INTEGER, "x", 15L, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDoubleStats(10.0, 100.0), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.FLOAT, "x", 15.0, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDoubleStats(10.0, 100.0), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "15", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDoubleStats(10.0, 100.0), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(15), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDoubleStats(10.0, 100.0), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.create(15), null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDoubleStats(10.0, 100.0), pred)); + } + + @Test + public void testPredEvalWithStringStats() throws Exception { + PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( + PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.INTEGER, "x", 100, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createStringStats("10", "1000"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.FLOAT, "x", 100.0, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createStringStats("10", "1000"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "100", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createStringStats("10", "1000"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(100), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createStringStats("10", "1000"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.create(100), null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createStringStats("10", "1000"), pred)); + } + + @Test + public void testPredEvalWithDateStats() throws Exception { + PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( + PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.INTEGER, "x", 15L, null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.FLOAT, "x", 15.0, null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "15", null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "1970-01-11", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "15.1", null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "__a15__1", null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "2000-01-16", null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "1970-01-16", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(15), null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(150), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.create(15), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDateStats(10, 100), pred)); + + } + + @Test + public void testPredEvalWithDecimalStats() throws Exception { + PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( + PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.INTEGER, "x", 15L, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDecimalStats("10.0", "100.0"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.FLOAT, "x", 15.0, null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDecimalStats("10.0", "100.0"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.STRING, "x", "15", null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDecimalStats("10.0", "100.0"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DATE, "x", new DateWritable(15), null); + assertEquals(TruthValue.NO, + RecordReaderImpl.evaluatePredicate(createDecimalStats("10.0", "100.0"), pred)); + + pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.create(15), null); + assertEquals(TruthValue.YES_NO, + RecordReaderImpl.evaluatePredicate(createDecimalStats("10.0", "100.0"), pred)); + + } + + @Test public void testEquals() throws Exception { PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf (PredicateLeaf.Operator.EQUALS, PredicateLeaf.Type.INTEGER, @@ -182,37 +376,6 @@ public void testEquals() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(0L, 10L), pred)); assertEquals(TruthValue.YES_NULL, RecordReaderImpl.evaluatePredicate(createIntStats(15L, 15L), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.EQUALS, - PredicateLeaf.Type.CHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.EQUALS, - PredicateLeaf.Type.VARCHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - } @Test @@ -232,36 +395,6 @@ public void testNullSafeEquals() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(0L, 10L), pred)); assertEquals(TruthValue.YES_NO, RecordReaderImpl.evaluatePredicate(createIntStats(15L, 15L), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, - PredicateLeaf.Type.CHAR, "x", "hello", null); - assertEquals(TruthValue.NO, - RecordReaderImpl.evaluatePredicate(createStringStats("world", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("hello", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "hello"), pred)); - assertEquals(TruthValue.NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "banana"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("hello", "hello"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, - PredicateLeaf.Type.VARCHAR, "x", "hello", null); - assertEquals(TruthValue.NO, - RecordReaderImpl.evaluatePredicate(createStringStats("world", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("hello", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "zombie"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "hello"), pred)); - assertEquals(TruthValue.NO, - RecordReaderImpl.evaluatePredicate(createStringStats("apple", "banana"), pred)); - assertEquals(TruthValue.YES_NO, - RecordReaderImpl.evaluatePredicate(createStringStats("hello", "hello"), pred)); } @Test @@ -279,36 +412,6 @@ public void testLessThan() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(10L, 15L), lessThan)); assertEquals(TruthValue.YES_NULL, RecordReaderImpl.evaluatePredicate(createIntStats(0L, 10L), lessThan)); - - PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( - PredicateLeaf.Operator.LESS_THAN, PredicateLeaf.Type.CHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.LESS_THAN, - PredicateLeaf.Type.VARCHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); } @Test @@ -326,36 +429,6 @@ public void testLessThanEquals() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(10L, 15L), pred)); assertEquals(TruthValue.YES_NULL, RecordReaderImpl.evaluatePredicate(createIntStats(0L, 10L), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, - PredicateLeaf.Type.CHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, - PredicateLeaf.Type.VARCHAR, "x", "b", null); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); } @Test @@ -374,39 +447,6 @@ public void testIn() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(10L, 30L), pred)); assertEquals(TruthValue.NO_NULL, RecordReaderImpl.evaluatePredicate(createIntStats(12L, 18L), pred)); - - args.clear(); - args.add("a"); - args.add("b"); - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.IN, - PredicateLeaf.Type.CHAR, "x", null, args); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.IN, - PredicateLeaf.Type.VARCHAR, "x", null, args); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); } @Test @@ -431,39 +471,6 @@ public void testBetween() throws Exception { RecordReaderImpl.evaluatePredicate(createIntStats(10L, 20L), pred)); assertEquals(TruthValue.YES_NULL, RecordReaderImpl.evaluatePredicate(createIntStats(12L, 18L), pred)); - - args.clear(); - args.add("a"); - args.add("b"); - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.BETWEEN, - PredicateLeaf.Type.CHAR, "x", null, args); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); - - pred = TestSearchArgumentImpl.createPredicateLeaf(PredicateLeaf.Operator.BETWEEN, - PredicateLeaf.Type.VARCHAR, "x", null, args); - assertEquals(TruthValue.NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("c", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "d"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "d"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "b"), pred)); - assertEquals(TruthValue.YES_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("a", "a"), pred)); - assertEquals(TruthValue.YES_NO_NULL, - RecordReaderImpl.evaluatePredicate(createStringStats("b", "b"), pred)); } @Test diff --git ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index 6a2a56a..b1524f7 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -27,13 +27,18 @@ import java.util.List; import java.util.Set; -import com.google.common.collect.Sets; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.ExpressionBuilder; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.ExpressionTree; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.junit.Test; +import com.google.common.collect.Sets; + /** * These test the SARG implementation. * The xml files were generated by setting hive.optimize.index.filter @@ -2810,4 +2815,36 @@ public void testBuilder() throws Exception { "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); } + + @Test + public void testBuilderComplexTypes() throws Exception { + SearchArgument sarg = + SearchArgument.FACTORY.newBuilder() + .startAnd() + .lessThan("x", new DateWritable(10)) + .lessThanEquals("y", new HiveChar("hi", 10)) + .equals("z", HiveDecimal.create("1.0")) + .end() + .build(); + assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" + + "leaf-1 = (LESS_THAN_EQUALS y hi)\n" + + "leaf-2 = (EQUALS z 1)\n" + + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); + + sarg = SearchArgument.FACTORY.newBuilder() + .startNot() + .startOr() + .isNull("x") + .between("y", HiveDecimal.create(10), 20.0) + .in("z", (byte)1, (short)2, (int)3) + .nullSafeEquals("a", new HiveVarchar("stinger", 100)) + .end() + .end() + .build(); + assertEquals("leaf-0 = (IS_NULL x)\n" + + "leaf-1 = (BETWEEN y 10 20.0)\n" + + "leaf-2 = (IN z 1 2 3)\n" + + "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" + + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); + } }