diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index f5023bb..f8a6e64 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.io.IOException; +import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.sql.Timestamp; @@ -2410,6 +2411,9 @@ static TruthValue evaluatePredicateRange(PredicateLeaf predicate, Object min, private static Object getBaseObjectForComparison(Object predObj, Object statsObj) { if (predObj != null) { + if (predObj instanceof ExprNodeConstantDesc) { + predObj = ((ExprNodeConstantDesc) predObj).getValue(); + } // following are implicitly convertible if (statsObj instanceof Long) { if (predObj instanceof Double) { @@ -2428,10 +2432,6 @@ private static Object getBaseObjectForComparison(Object predObj, Object statsObj return Double.valueOf(predObj.toString()); } } else if (statsObj instanceof String) { - // Ex: where d = date '1970-02-01' will be ExprNodeConstantDesc - if (predObj instanceof ExprNodeConstantDesc) { - return ((ExprNodeConstantDesc) predObj).getValue().toString(); - } return predObj.toString(); } else if (statsObj instanceof HiveDecimal) { if (predObj instanceof Long) { @@ -2440,6 +2440,8 @@ private static Object getBaseObjectForComparison(Object predObj, Object statsObj return HiveDecimal.create(predObj.toString()); } else if (predObj instanceof String) { return HiveDecimal.create(predObj.toString()); + } else if (predObj instanceof BigDecimal) { + return HiveDecimal.create((BigDecimal)predObj); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java index 2c53f65..84d7e0f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -18,14 +18,6 @@ package org.apache.hadoop.hive.ql.io.sarg; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -57,6 +49,15 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import java.math.BigDecimal; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * The implementation of SearchArguments. */ @@ -947,7 +948,8 @@ private static Object boxLiteral(Object literal) { literal instanceof Long || literal instanceof Double || literal instanceof DateWritable || - literal instanceof HiveDecimal) { + literal instanceof HiveDecimal || + literal instanceof BigDecimal) { return literal; } else if (literal instanceof HiveChar || literal instanceof HiveVarchar) { @@ -979,7 +981,8 @@ private static Object boxLiteral(Object literal) { return PredicateLeaf.Type.FLOAT; } else if (literal instanceof DateWritable) { return PredicateLeaf.Type.DATE; - } else if (literal instanceof HiveDecimal) { + } else if (literal instanceof HiveDecimal || + literal instanceof BigDecimal) { return PredicateLeaf.Type.DECIMAL; } throw new IllegalArgumentException("Unknown type for literal " + literal); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index b1524f7..a4ce32e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -18,15 +18,7 @@ package org.apache.hadoop.hive.ql.io.sarg; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertTrue; - -import java.beans.XMLDecoder; -import java.io.ByteArrayInputStream; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Set; - +import com.google.common.collect.Sets; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; @@ -37,7 +29,15 @@ import org.apache.hadoop.hive.serde2.io.DateWritable; import org.junit.Test; -import com.google.common.collect.Sets; +import java.beans.XMLDecoder; +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.util.List; +import java.util.Set; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; /** * These test the SARG implementation. @@ -2847,4 +2847,36 @@ public void testBuilderComplexTypes() throws Exception { "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); } + + @Test + public void testBuilderComplexTypes2() throws Exception { + SearchArgument sarg = + SearchArgument.FACTORY.newBuilder() + .startAnd() + .lessThan("x", new DateWritable(10)) + .lessThanEquals("y", new HiveChar("hi", 10)) + .equals("z", new BigDecimal("1.0")) + .end() + .build(); + assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" + + "leaf-1 = (LESS_THAN_EQUALS y hi)\n" + + "leaf-2 = (EQUALS z 1.0)\n" + + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); + + sarg = SearchArgument.FACTORY.newBuilder() + .startNot() + .startOr() + .isNull("x") + .between("y", new BigDecimal(10), 20.0) + .in("z", (byte)1, (short)2, (int)3) + .nullSafeEquals("a", new HiveVarchar("stinger", 100)) + .end() + .end() + .build(); + assertEquals("leaf-0 = (IS_NULL x)\n" + + "leaf-1 = (BETWEEN y 10 20.0)\n" + + "leaf-2 = (IN z 1 2 3)\n" + + "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" + + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); + } } diff --git ql/src/test/queries/clientpositive/orc_ppd_decimal.q ql/src/test/queries/clientpositive/orc_ppd_decimal.q index a93590e..a54b862 100644 --- ql/src/test/queries/clientpositive/orc_ppd_decimal.q +++ ql/src/test/queries/clientpositive/orc_ppd_decimal.q @@ -85,6 +85,18 @@ set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc where d<=cast('11.22' as float); set hive.optimize.index.filter=false; +select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal); + +set hive.optimize.index.filter=true; +select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal); + +set hive.optimize.index.filter=false; +select sum(hash(*)) from newtypesorc where d<=11.22BD; + +set hive.optimize.index.filter=true; +select sum(hash(*)) from newtypesorc where d<=11.22BD; + +set hive.optimize.index.filter=false; select sum(hash(*)) from newtypesorc where d<=12; set hive.optimize.index.filter=true; diff --git ql/src/test/results/clientpositive/orc_ppd_decimal.q.out ql/src/test/results/clientpositive/orc_ppd_decimal.q.out index 0c11ea8..6ddff10 100644 --- ql/src/test/results/clientpositive/orc_ppd_decimal.q.out +++ ql/src/test/results/clientpositive/orc_ppd_decimal.q.out @@ -254,6 +254,42 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### 81475875500 +PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal) +PREHOOK: type: QUERY +PREHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +-252951929000 +PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal) +PREHOOK: type: QUERY +PREHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +-252951929000 +PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD +PREHOOK: type: QUERY +PREHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +81475875500 +PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD +PREHOOK: type: QUERY +PREHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@newtypesorc +#### A masked pattern was here #### +81475875500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=12 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc