diff --git ql/src/test/queries/clientnegative/decimal_precision.q ql/src/test/queries/clientnegative/decimal_precision.q new file mode 100644 index 0000000..2d04077 --- /dev/null +++ ql/src/test/queries/clientnegative/decimal_precision.q @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS DECIMAL_PRECISION; + +CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +SELECT dec * 123456789012345678901234567890.1234567bd FROM DECIMAL_PRECISION; + +DROP TABLE DECIMAL_PRECISION; diff --git ql/src/test/queries/clientnegative/decimal_precision_1.q ql/src/test/queries/clientnegative/decimal_precision_1.q new file mode 100644 index 0000000..f970298 --- /dev/null +++ ql/src/test/queries/clientnegative/decimal_precision_1.q @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS DECIMAL_PRECISION; + +CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +SELECT * from DECIMAL_PRECISION WHERE dec > 1234567890123456789.01234567bd; + +DROP TABLE DECIMAL_PRECISION; diff --git ql/src/test/queries/clientpositive/decimal_precision.q ql/src/test/queries/clientpositive/decimal_precision.q index 7744f40..dd34737 100644 --- ql/src/test/queries/clientpositive/decimal_precision.q +++ ql/src/test/queries/clientpositive/decimal_precision.q @@ -17,4 +17,11 @@ SELECT dec, dec * dec FROM DECIMAL_PRECISION ORDER BY dec; SELECT avg(dec), sum(dec) FROM DECIMAL_PRECISION; +SELECT dec * cast('123456789012345678901234567890.1234567' as decimal) FROM DECIMAL_PRECISION LIMIT 1; +SELECT * from DECIMAL_PRECISION WHERE dec > cast('123456789012345678901234567890.1234567' as decimal) LIMIT 1; +SELECT dec * 123456789012345678901234567890.1234567 FROM DECIMAL_PRECISION LIMIT 1; + +SELECT MIN(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION; +SELECT COUNT(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION; + DROP TABLE DECIMAL_PRECISION; diff --git ql/src/test/queries/clientpositive/decimal_udf.q ql/src/test/queries/clientpositive/decimal_udf.q index 9ddd47a..b5ff088 100644 --- ql/src/test/queries/clientpositive/decimal_udf.q +++ ql/src/test/queries/clientpositive/decimal_udf.q @@ -113,4 +113,16 @@ SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value; EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF; SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF; +-- min +EXPLAIN SELECT MIN(key) FROM DECIMAL_UDF; +SELECT MIN(key) FROM DECIMAL_UDF; + +-- max +EXPLAIN SELECT MAX(key) FROM DECIMAL_UDF; +SELECT MAX(key) FROM DECIMAL_UDF; + +-- count +EXPLAIN SELECT COUNT(key) FROM DECIMAL_UDF; +SELECT COUNT(key) FROM DECIMAL_UDF; + DROP TABLE IF EXISTS DECIMAL_UDF; diff --git ql/src/test/results/clientnegative/decimal_precision.q.out ql/src/test/results/clientnegative/decimal_precision.q.out new file mode 100644 index 0000000..1eef00f --- /dev/null +++ ql/src/test/results/clientnegative/decimal_precision.q.out @@ -0,0 +1,16 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_PRECISION +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_PRECISION +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@DECIMAL_PRECISION +FAILED: SemanticException [Error 10029]: Line 3:13 Invalid numerical constant '123456789012345678901234567890.1234567bd' diff --git ql/src/test/results/clientnegative/decimal_precision_1.q.out ql/src/test/results/clientnegative/decimal_precision_1.q.out new file mode 100644 index 0000000..f489223 --- /dev/null +++ ql/src/test/results/clientnegative/decimal_precision_1.q.out @@ -0,0 +1,16 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_PRECISION +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_PRECISION +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE DECIMAL_PRECISION(dec decimal) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@DECIMAL_PRECISION +FAILED: SemanticException [Error 10029]: Line 3:44 Invalid numerical constant '1234567890123456789.01234567bd' diff --git ql/src/test/results/clientpositive/decimal_precision.q.out ql/src/test/results/clientpositive/decimal_precision.q.out index 2348869..1171614 100644 --- ql/src/test/results/clientpositive/decimal_precision.q.out +++ ql/src/test/results/clientpositive/decimal_precision.q.out @@ -514,6 +514,50 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision #### A masked pattern was here #### NULL NULL +PREHOOK: query: SELECT dec * cast('123456789012345678901234567890.1234567' as decimal) FROM DECIMAL_PRECISION LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +POSTHOOK: query: SELECT dec * cast('123456789012345678901234567890.1234567' as decimal) FROM DECIMAL_PRECISION LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT * from DECIMAL_PRECISION WHERE dec > cast('123456789012345678901234567890.1234567' as decimal) LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +POSTHOOK: query: SELECT * from DECIMAL_PRECISION WHERE dec > cast('123456789012345678901234567890.1234567' as decimal) LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +PREHOOK: query: SELECT dec * 123456789012345678901234567890.1234567 FROM DECIMAL_PRECISION LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +POSTHOOK: query: SELECT dec * 123456789012345678901234567890.1234567 FROM DECIMAL_PRECISION LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT MIN(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +POSTHOOK: query: SELECT MIN(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT COUNT(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(cast('123456789012345678901234567890.1234567' as decimal)) FROM DECIMAL_PRECISION +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision +#### A masked pattern was here #### +0 PREHOOK: query: DROP TABLE DECIMAL_PRECISION PREHOOK: type: DROPTABLE PREHOOK: Input: default@decimal_precision diff --git ql/src/test/results/clientpositive/decimal_udf.q.out ql/src/test/results/clientpositive/decimal_udf.q.out index 5bd52c9..52c8a81 100644 --- ql/src/test/results/clientpositive/decimal_udf.q.out +++ ql/src/test/results/clientpositive/decimal_udf.q.out @@ -2482,6 +2482,216 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_udf #### A masked pattern was here #### [{"x":-1.2345678901234567E9,"y":1.0},{"x":-148.75058823529412,"y":34.0},{"x":1.2345678901234567E9,"y":1.0}] +PREHOOK: query: -- min +EXPLAIN SELECT MIN(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- min +EXPLAIN SELECT MIN(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION MIN (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: key + type: decimal + outputColumnNames: key + Group By Operator + aggregations: + expr: min(key) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: decimal + Reduce Operator Tree: + Group By Operator + aggregations: + expr: min(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: _col0 + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT MIN(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT MIN(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-1234567890.123456789 +PREHOOK: query: -- max +EXPLAIN SELECT MAX(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- max +EXPLAIN SELECT MAX(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION MAX (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: key + type: decimal + outputColumnNames: key + Group By Operator + aggregations: + expr: max(key) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: decimal + Reduce Operator Tree: + Group By Operator + aggregations: + expr: max(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: _col0 + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT MAX(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT MAX(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +1234567890.12345678 +PREHOOK: query: -- count +EXPLAIN SELECT COUNT(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- count +EXPLAIN SELECT COUNT(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION COUNT (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: key + type: decimal + outputColumnNames: key + Group By Operator + aggregations: + expr: count(key) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: bigint + Reduce Operator Tree: + Group By Operator + aggregations: + expr: count(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: _col0 + type: bigint + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT COUNT(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +36 PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF PREHOOK: type: DROPTABLE PREHOOK: Input: default@decimal_udf diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index 16afd72..742493c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -276,10 +276,14 @@ public Object convert(Object input) { if (input == null) { return null; } - return outputOI.set(r, PrimitiveObjectInspectorUtils.getHiveDecimal(input, - inputOI)); + + try { + return outputOI.set(r, PrimitiveObjectInspectorUtils.getHiveDecimal(input, + inputOI)); + } catch (NumberFormatException e) { + return null; + } } - } public static class BinaryConverter implements Converter{ diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index e7d890f..94062dd 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -778,52 +778,48 @@ public static HiveDecimal getHiveDecimal(Object o, PrimitiveObjectInspector oi) } HiveDecimal result = null; - try { - switch (oi.getPrimitiveCategory()) { - case VOID: - result = null; - break; - case BOOLEAN: - result = ((BooleanObjectInspector) oi).get(o) ? - HiveDecimal.ONE : HiveDecimal.ZERO; - break; - case BYTE: - result = new HiveDecimal(((ByteObjectInspector) oi).get(o)); - break; - case SHORT: - result = new HiveDecimal(((ShortObjectInspector) oi).get(o)); - break; - case INT: - result = new HiveDecimal(((IntObjectInspector) oi).get(o)); - break; - case LONG: - result = new HiveDecimal(((LongObjectInspector) oi).get(o)); - break; - case FLOAT: - Float f = ((FloatObjectInspector) oi).get(o); - result = new HiveDecimal(f.toString()); - break; - case DOUBLE: - Double d = ((DoubleObjectInspector) oi).get(o); - result = new HiveDecimal(d.toString()); - break; - case STRING: - result = new HiveDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); - break; - case TIMESTAMP: - Double ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) + switch (oi.getPrimitiveCategory()) { + case VOID: + result = null; + break; + case BOOLEAN: + result = ((BooleanObjectInspector) oi).get(o) ? + HiveDecimal.ONE : HiveDecimal.ZERO; + break; + case BYTE: + result = new HiveDecimal(((ByteObjectInspector) oi).get(o)); + break; + case SHORT: + result = new HiveDecimal(((ShortObjectInspector) oi).get(o)); + break; + case INT: + result = new HiveDecimal(((IntObjectInspector) oi).get(o)); + break; + case LONG: + result = new HiveDecimal(((LongObjectInspector) oi).get(o)); + break; + case FLOAT: + Float f = ((FloatObjectInspector) oi).get(o); + result = new HiveDecimal(f.toString()); + break; + case DOUBLE: + Double d = ((DoubleObjectInspector) oi).get(o); + result = new HiveDecimal(d.toString()); + break; + case STRING: + result = new HiveDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + case TIMESTAMP: + Double ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); - result = new HiveDecimal(ts.toString()); - break; - case DECIMAL: - result = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); - break; - default: - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } - } catch(NumberFormatException e) { - // return null + result = new HiveDecimal(ts.toString()); + break; + case DECIMAL: + result = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + break; + default: + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); } return result; }