diff --git data/files/datatypes.txt data/files/datatypes.txt index 87e0573..66ef826 100644 --- data/files/datatypes.txt +++ data/files/datatypes.txt @@ -1,3 +1,3 @@ -\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N --1false-1.1\N\N\N-1-1-1.0-1\N\N\N -1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789 +\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N +-1false-1.1\N\N\N-1-1-1.0-1\N\N\N\N +1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd22012-04-22 09:00:00.123456789123456789.0123456 diff --git data/files/kv7.txt data/files/kv7.txt new file mode 100644 index 0000000..b83c460 --- /dev/null +++ data/files/kv7.txt @@ -0,0 +1,38 @@ +-4400 4400 +1E+99 0 +1E-99 0 +0 0 +100 100 +10 10 +1 1 +0.1 0 +0.01 0 +200 200 +20 20 +2 2 +0 0 +0.2 0 +0.02 0 +0.3 0 +0.33 0 +0.333 0 +-0.3 0 +-0.33 0 +-0.333 0 +1.0 1 +2 2 +3.14 3 +-1.12 -1 +-1.12 -1 +-1.122 -11 +1.12 1 +1.122 1 +124.00 124 +125.2 125 +-1255.49 -1255 +3.14 3 +3.14 3 +3.140 4 +0.9999999999999999999999999 1 +-1234567890.1234567890 -1234567890 +1234567890.1234567800 1234567890 diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java index 3bfff92..bebcbda 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; +import java.math.MathContext; import java.net.URL; import java.sql.Array; import java.sql.Blob; @@ -102,19 +103,27 @@ public InputStream getAsciiStream(String columnName) throws SQLException { } public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - throw new SQLException("Method not supported"); + Object obj = getObject(columnIndex); + if (obj == null) { + return null; + } + if (obj instanceof BigDecimal) { + return ((BigDecimal) obj); + } + throw new SQLException("Cannot convert column " + columnIndex + " to BigDecimal"); } public BigDecimal getBigDecimal(String columnName) throws SQLException { - throw new SQLException("Method not supported"); + return getBigDecimal(findColumn(columnName)); } public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - throw new SQLException("Method not supported"); + MathContext mc = new MathContext(scale); + return getBigDecimal(columnIndex).round(mc); } public BigDecimal getBigDecimal(String columnName, int scale) throws SQLException { - throw new SQLException("Method not supported"); + return getBigDecimal(findColumn(columnName), scale); } public InputStream getBinaryStream(int columnIndex) throws SQLException { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java index 0121a66..48b2e86 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java @@ -112,6 +112,8 @@ public String getColumnTypeName(int column) throws SQLException { return serdeConstants.BIGINT_TYPE_NAME; } else if ("timestamp".equalsIgnoreCase(type)) { return serdeConstants.TIMESTAMP_TYPE_NAME; + } else if ("decimal".equalsIgnoreCase(type)) { + return serdeConstants.DECIMAL_TYPE_NAME; } else if (type.startsWith("map<")) { return serdeConstants.STRING_TYPE_NAME; } else if (type.startsWith("array<")) { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java index c33b346..920a4ba 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java @@ -82,6 +82,8 @@ static int columnDisplaySize(int columnType) throws SQLException { // see http://download.oracle.com/javase/6/docs/api/constant-values.html#java.lang.Double.MAX_EXPONENT case Types.DOUBLE: return 25; // e.g. -(17#).e-#### + case Types.DECIMAL: + return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } @@ -108,6 +110,8 @@ static int columnPrecision(int columnType) throws SQLException { return 15; case Types.TIMESTAMP: return 29; + case Types.DECIMAL: + return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } @@ -129,6 +133,8 @@ static int columnScale(int columnType) throws SQLException { return 15; case Types.TIMESTAMP: return 9; + case Types.DECIMAL: + return Integer.MAX_VALUE; default: throw new SQLException("Invalid column type: " + columnType); } @@ -153,6 +159,8 @@ public Integer getNumPrecRadix() { return 10; } else if (type.equalsIgnoreCase("bigint")) { return 10; + } else if (type.equalsIgnoreCase("decimal")) { + return 10; } else if (type.equalsIgnoreCase("float")) { return 2; } else if (type.equalsIgnoreCase("double")) { diff --git jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java index 595a1ed..c93d00b 100644 --- jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java @@ -48,6 +48,8 @@ public static int hiveTypeToSqlType(String type) throws SQLException { return Types.BIGINT; } else if ("timestamp".equalsIgnoreCase(type)) { return Types.TIMESTAMP; + } else if ("decimal".equalsIgnoreCase(type)) { + return Types.DECIMAL; } else if (type.startsWith("map<")) { return Types.VARCHAR; } else if (type.startsWith("array<")) { diff --git jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java index 1c59cb3..639418f 100644 --- jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java +++ jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java @@ -148,7 +148,8 @@ protected void setUp() throws Exception { + " c14 map>," + " c15 struct>," + " c16 array,n:int>>," - + " c17 timestamp) comment '"+dataTypeTableComment + + " c17 timestamp, " + + " c18 decimal) comment'" + dataTypeTableComment +"' partitioned by (dt STRING)"); assertFalse(res.next()); @@ -381,6 +382,7 @@ public void testDataTypes() throws Exception { assertEquals("[]", res.getString(16)); assertEquals(null, res.getString(17)); assertEquals(null, res.getTimestamp(17)); + assertEquals(null, res.getBigDecimal(18)); // row 3 assertTrue(res.next()); @@ -402,6 +404,7 @@ public void testDataTypes() throws Exception { assertEquals("[[{}, 1], [{c=d, a=b}, 2]]", res.getString(16)); assertEquals("2012-04-22 09:00:00.123456789", res.getString(17)); assertEquals("2012-04-22 09:00:00.123456789", res.getTimestamp(17).toString()); + assertEquals("123456789.0123456", res.getBigDecimal(18).toString()); // test getBoolean rules on non-boolean columns assertEquals(true, res.getBoolean(1)); @@ -803,13 +806,13 @@ public void testResultSetMetaData() throws SQLException { ResultSet res = stmt.executeQuery( "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " + - "c1*2, sentences(null, null, null) as b, c17 from " + dataTypeTableName + " limit 1"); + "c1*2, sentences(null, null, null) as b, c17, c18 from " + dataTypeTableName + " limit 1"); ResultSetMetaData meta = res.getMetaData(); ResultSet colRS = con.getMetaData().getColumns(null, null, dataTypeTableName.toLowerCase(), null); - assertEquals(15, meta.getColumnCount()); + assertEquals(16, meta.getColumnCount()); assertTrue(colRS.next()); @@ -1012,6 +1015,13 @@ public void testResultSetMetaData() throws SQLException { assertEquals(29, meta.getPrecision(15)); assertEquals(9, meta.getScale(15)); + assertEquals("c18", meta.getColumnName(16)); + assertEquals(Types.DECIMAL, meta.getColumnType(16)); + assertEquals("decimal", meta.getColumnTypeName(16)); + assertEquals(Integer.MAX_VALUE, meta.getColumnDisplaySize(16)); + assertEquals(Integer.MAX_VALUE, meta.getPrecision(16)); + assertEquals(Integer.MAX_VALUE, meta.getScale(16)); + for (int i = 1; i <= meta.getColumnCount(); i++) { assertFalse(meta.isAutoIncrement(i)); assertFalse(meta.isCurrency(i)); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 3d89e4c..bc365cb 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -395,7 +395,7 @@ public static void setSerdeParam(SerDeInfo sdi, Properties schema, org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME, "string"); typeToThriftTypeMap.put( org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME, "binary"); - // These 3 types are not supported yet. + // These 4 types are not supported yet. // We should define a complex type date in thrift that contains a single int // member, and DynamicSerDe // should convert it to date type at runtime. @@ -406,6 +406,8 @@ public static void setSerdeParam(SerDeInfo sdi, Properties schema, typeToThriftTypeMap .put(org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME, "timestamp"); + typeToThriftTypeMap.put( + org.apache.hadoop.hive.serde.serdeConstants.DECIMAL_TYPE_NAME, "decimal"); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index ab9b4e9..0c60447 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -200,6 +201,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTranslate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion; @@ -403,6 +405,8 @@ GenericUDFTimestamp.class); registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, GenericUDFToBinary.class); + registerGenericUDF(serdeConstants.DECIMAL_TYPE_NAME, + GenericUDFToDecimal.class); // Aggregate functions registerGenericUDAF("max", new GenericUDAFMax()); @@ -638,7 +642,8 @@ static void registerNumericType(String typeName, int level) { registerNumericType(serdeConstants.BIGINT_TYPE_NAME, 4); registerNumericType(serdeConstants.FLOAT_TYPE_NAME, 5); registerNumericType(serdeConstants.DOUBLE_TYPE_NAME, 6); - registerNumericType(serdeConstants.STRING_TYPE_NAME, 7); + registerNumericType(serdeConstants.DECIMAL_TYPE_NAME, 7); + registerNumericType(serdeConstants.STRING_TYPE_NAME, 8); } /** @@ -720,6 +725,11 @@ public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { && to.equals(TypeInfoFactory.doubleTypeInfo)) { return true; } + // Allow implicit String to Decimal conversion + if (from.equals(TypeInfoFactory.stringTypeInfo) + && to.equals(TypeInfoFactory.decimalTypeInfo)) { + return true; + } // Void can be converted to any type if (from.equals(TypeInfoFactory.voidTypeInfo)) { return true; @@ -731,7 +741,7 @@ public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { } // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double - // -> String + // Decimal -> String Integer f = numericTypes.get(from); Integer t = numericTypes.get(to); if (f == null || t == null) { @@ -1020,8 +1030,52 @@ public static Method getMethodInternal(Class udfClass, List mlist, bo throw new NoMatchingMethodException(udfClass, argumentsPassed, mlist); } if (udfMethods.size() > 1) { - // Ambiguous method found - throw new AmbiguousMethodException(udfClass, argumentsPassed, mlist); + + // if the only difference is numeric types, pick the method + // with the smallest overall numeric types. + int lowestNumericType = Integer.MAX_VALUE; + boolean multiple = true; + Method candidate = null; + List referenceArguments = null; + + for (Method m: udfMethods) { + int maxNumericType = 0; + + List argumentsAccepted = TypeInfoUtils.getParameterTypeInfos(m, argumentsPassed.size()); + + if (referenceArguments == null) { + referenceArguments = argumentsAccepted; + } + + Iterator referenceIterator = referenceArguments.iterator(); + + for (TypeInfo accepted: argumentsAccepted) { + TypeInfo reference = referenceIterator.next(); + + if (numericTypes.containsKey(accepted)) { + int typeValue = numericTypes.get(accepted); + maxNumericType = typeValue > maxNumericType ? typeValue : maxNumericType; + } else if (!accepted.equals(reference)) { + // Ambiguous method found + throw new AmbiguousMethodException(udfClass, argumentsPassed, mlist); + } + } + + if (lowestNumericType > maxNumericType) { + multiple = false; + lowestNumericType = maxNumericType; + candidate = m; + } else if (maxNumericType == lowestNumericType) { + multiple = true; + } + } + + if (!multiple) { + return candidate; + } else { + // Ambiguous method found + throw new AmbiguousMethodException(udfClass, argumentsPassed, mlist); + } } return udfMethods.get(0); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java index 5742797..43984af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java @@ -66,12 +66,21 @@ public Method getEvalMethod(List argTypeInfos) throws UDFArgumentExcep List pTypeInfos = null; List modArgTypeInfos = new ArrayList(); - // If either argument is a string, we convert to a double because a number - // in string form should always be convertible into a double + // If either argument is a string, we convert to a double or decimal because a number + // in string form should always be convertible into either of those if (argTypeInfos.get(0).equals(TypeInfoFactory.stringTypeInfo) || argTypeInfos.get(1).equals(TypeInfoFactory.stringTypeInfo)) { - modArgTypeInfos.add(TypeInfoFactory.doubleTypeInfo); - modArgTypeInfos.add(TypeInfoFactory.doubleTypeInfo); + + // Default is double, but if one of the sides is already in decimal we + // complete the operation in that type. + if (argTypeInfos.get(0).equals(TypeInfoFactory.decimalTypeInfo) + || argTypeInfos.get(1).equals(TypeInfoFactory.decimalTypeInfo)) { + modArgTypeInfos.add(TypeInfoFactory.decimalTypeInfo); + modArgTypeInfos.add(TypeInfoFactory.decimalTypeInfo); + } else { + modArgTypeInfos.add(TypeInfoFactory.doubleTypeInfo); + modArgTypeInfos.add(TypeInfoFactory.doubleTypeInfo); + } } else { // If it's a void, we change the type to a byte because once the types // are run through getCommonClass(), a byte and any other type T will diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 3f4173c..ff4a6f2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -149,6 +149,7 @@ TokenToTypeName.put(HiveParser.TOK_DATE, serdeConstants.DATE_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DATETIME, serdeConstants.DATETIME_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME); + TokenToTypeName.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); } public static String getTypeName(int token) throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g index 1b486ab..4616eac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g @@ -108,6 +108,7 @@ TOK_DATETIME; TOK_TIMESTAMP; TOK_STRING; TOK_BINARY; +TOK_DECIMAL; TOK_LIST; TOK_STRUCT; TOK_MAP; @@ -1495,6 +1496,7 @@ primitiveType | KW_TIMESTAMP -> TOK_TIMESTAMP | KW_STRING -> TOK_STRING | KW_BINARY -> TOK_BINARY + | KW_DECIMAL -> TOK_DECIMAL ; listType @@ -2463,6 +2465,7 @@ KW_DOUBLE: 'DOUBLE'; KW_DATE: 'DATE'; KW_DATETIME: 'DATETIME'; KW_TIMESTAMP: 'TIMESTAMP'; +KW_DECIMAL: 'DECIMAL'; KW_STRING: 'STRING'; KW_ARRAY: 'ARRAY'; KW_STRUCT: 'STRUCT'; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 38e4a4f..c267ad5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -501,6 +501,8 @@ public static ColumnExprProcessor getColumnExprProcessor() { serdeConstants.BINARY_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME); + conversionFunctionTextHashMap.put(HiveParser.TOK_DECIMAL, + serdeConstants.DECIMAL_TYPE_NAME); } public static boolean isRedundantConversionFunction(ASTNode expr, diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java index 41043bc..bfce482 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -38,6 +39,7 @@ private final DoubleWritable resultDouble = new DoubleWritable(); private final LongWritable resultLong = new LongWritable(); private final IntWritable resultInt = new IntWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public DoubleWritable evaluate(DoubleWritable n) { if (n == null) { @@ -68,4 +70,13 @@ public IntWritable evaluate(IntWritable n) { return resultInt; } + + public BigDecimalWritable evaluate(BigDecimalWritable n) { + if (n == null) { + return null; + } + + resultBigDecimal.set(n.getBigDecimal().abs()); + return resultBigDecimal; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java index 63d0255..14c16ec 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.NumericOpMethodResolver; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -48,6 +49,7 @@ public UDFBaseNumericOp() { protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a, ByteWritable b); @@ -61,4 +63,5 @@ public UDFBaseNumericOp() { public abstract DoubleWritable evaluate(DoubleWritable a, DoubleWritable b); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java index b220805..cb7dca4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -45,6 +46,7 @@ public UDFBaseNumericUnaryOp() { protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a); @@ -58,4 +60,5 @@ public UDFBaseNumericUnaryOp() { public abstract DoubleWritable evaluate(DoubleWritable a); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java index 01dd4d6..927c8b7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ + " 0\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFCeil extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public UDFCeil() { } @@ -48,4 +54,14 @@ public LongWritable evaluate(DoubleWritable i) { } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + int origScale = bd.scale(); + bigDecimalWritable.set(bd.setScale(0, BigDecimal.ROUND_CEILING).setScale(origScale)); + return bigDecimalWritable; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java index 510a161..fc07137 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ + " -1\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFFloor extends UDF { - private LongWritable result = new LongWritable(); + private final LongWritable result = new LongWritable(); + private final BigDecimalWritable bdResult = new BigDecimalWritable(); public UDFFloor() { } @@ -48,4 +54,14 @@ public LongWritable evaluate(DoubleWritable i) { } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + int origScale = bd.scale(); + bdResult.set(bd.setScale(0, BigDecimal.ROUND_FLOOR).setScale(origScale)); + return bdResult; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java index 0455aa9..a8488d4 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -29,12 +33,15 @@ @Description(name = "/", value = "a _FUNC_ b - Divide a by b", extended = "Example:\n" + " > SELECT 3 _FUNC_ 2 FROM src LIMIT 1;\n" + " 1.5") /** - * Note that in SQL, the return type of divide is not necessarily the same + * Note that in SQL, the return type of divide is not necessarily the same * as the parameters. For example, 3 / 2 = 1.5, not 1. To follow SQL, we always * return a double for divide. */ public class UDFOPDivide extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); + + private final int MAX_SCALE = 65; // max compatible with MySQL public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":" @@ -46,4 +53,18 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { doubleWritable.set(a.get() / b.get()); return doubleWritable; } + + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + if (b.getBigDecimal().compareTo(BigDecimal.ZERO) == 0) { + return null; + } else { + bigDecimalWritable.set(a.getBigDecimal().divide( + b.getBigDecimal(), MAX_SCALE, RoundingMode.HALF_UP)); + } + + return bigDecimalWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java index 8ed1cc6..f884b9a 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,15 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { doubleWritable.set(a.get() - b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().subtract(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java index 1935f03..9948c1f 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +110,21 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { doubleWritable.set(a.get() % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + BigDecimal av = a.getBigDecimal(); + BigDecimal bv = b.getBigDecimal(); + + if (bv.compareTo(BigDecimal.ZERO) == 0) { + return null; + } + + bigDecimalWritable.set(av.remainder(bv)); + return bigDecimalWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java index ce2c515..9058651 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,14 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { doubleWritable.set(a.get() * b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().multiply(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java index 2378df2..3c14fef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -90,4 +91,13 @@ public DoubleWritable evaluate(DoubleWritable a) { return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } + bigDecimalWritable.set(a.getBigDecimal().negate()); + return bigDecimalWritable; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java index 705c6eb..5722d8b 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -113,4 +114,14 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().add(b.getBigDecimal())); + return bigDecimalWritable; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java index c2c45e4..0711890 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -66,4 +67,9 @@ public DoubleWritable evaluate(DoubleWritable a) { return a; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + return a; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java index 3b86e9c..85531a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -108,4 +111,21 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { doubleWritable.set(((a.get() % b.get()) + b.get()) % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + BigDecimal av = a.getBigDecimal(); + BigDecimal bv = b.getBigDecimal(); + + if (bv.compareTo(BigDecimal.ZERO) == 0) { + return null; + } + + bigDecimalWritable.set(av.remainder(bv).add(bv).remainder(bv)); + return bigDecimalWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java index 197adbb..6795668 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java @@ -20,7 +20,9 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.io.IntWritable; /** * UDFPower. @@ -31,7 +33,8 @@ extended = "Example:\n" + " > SELECT _FUNC_(2, 3) FROM src LIMIT 1;\n" + " 8") public class UDFPower extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable resultDouble = new DoubleWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public UDFPower() { } @@ -43,8 +46,32 @@ public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { if (a == null || b == null) { return null; } else { - result.set(Math.pow(a.get(), b.get())); - return result; + resultDouble.set(Math.pow(a.get(), b.get())); + return resultDouble; + } + } + + /** + * Raise a to the power of b. + */ + public DoubleWritable evaluate(DoubleWritable a, IntWritable b) { + if (a == null || b == null) { + return null; + } else { + resultDouble.set(Math.pow(a.get(), b.get())); + return resultDouble; + } + } + + /** + * Raise a to the power of b + */ + public BigDecimalWritable evaluate(BigDecimalWritable a, IntWritable b) { + if (a == null || b == null) { + return null; + } else { + resultBigDecimal.set(a.getBigDecimal().pow(b.get())); + return resultBigDecimal; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java index 892c0d3..af8186a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; @@ -35,6 +36,7 @@ extended = "Example:\n" + " > SELECT _FUNC_(12.3456, 1) FROM src LIMIT 1;\n" + " 12.3'") public class UDFRound extends UDF { + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); private final DoubleWritable doubleWritable = new DoubleWritable(); public UDFRound() { @@ -65,4 +67,26 @@ public DoubleWritable evaluate(DoubleWritable n, IntWritable i) { return evaluate(n, i.get()); } + private BigDecimalWritable evaluate(BigDecimalWritable n, int i) { + if (n == null) { + return null; + } + BigDecimal bd = n.getBigDecimal(); + int origScale = bd.scale(); + bd = n.getBigDecimal().setScale(i, RoundingMode.HALF_UP).setScale(origScale); + bigDecimalWritable.set(bd); + return bigDecimalWritable; + } + + public BigDecimalWritable evaluate(BigDecimalWritable n) { + return evaluate(n, 0); + } + + public BigDecimalWritable evaluate(BigDecimalWritable n, IntWritable i) { + if (i == null) { + return null; + } + return evaluate(n, i.get()); + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index f3afd33..0d98551 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +38,7 @@ * */ public class UDFToBoolean extends UDF { - private BooleanWritable booleanWritable = new BooleanWritable(); + private final BooleanWritable booleanWritable = new BooleanWritable(); public UDFToBoolean() { } @@ -172,4 +175,13 @@ public BooleanWritable evaluate(TimestampWritable i) { } } + public BooleanWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + booleanWritable.set(BigDecimal.ZERO.compareTo(i.getBigDecimal()) != 0); + return booleanWritable; + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index 1b3b744..c5830ea 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToByte extends UDF { - private ByteWritable byteWritable = new ByteWritable(); + private final ByteWritable byteWritable = new ByteWritable(); public UDFToByte() { } @@ -181,4 +182,12 @@ public ByteWritable evaluate(TimestampWritable i) { } } + public ByteWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + byteWritable.set(i.getBigDecimal().byteValue()); + return byteWritable; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java index ce4660c..c57e31e 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -34,7 +35,7 @@ * */ public class UDFToDouble extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); public UDFToDouble() { } @@ -183,4 +184,12 @@ public DoubleWritable evaluate(TimestampWritable i) { } } + public DoubleWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + doubleWritable.set(i.getBigDecimal().doubleValue()); + return doubleWritable; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java index c6b197e..61591e9 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +36,7 @@ * */ public class UDFToFloat extends UDF { - private FloatWritable floatWritable = new FloatWritable(); + private final FloatWritable floatWritable = new FloatWritable(); public UDFToFloat() { } @@ -184,4 +185,13 @@ public FloatWritable evaluate(TimestampWritable i) { } } + public FloatWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + floatWritable.set(i.getBigDecimal().floatValue()); + return floatWritable; + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index 9b9d7df..018b3de 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToInteger extends UDF { - private IntWritable intWritable = new IntWritable(); + private final IntWritable intWritable = new IntWritable(); public UDFToInteger() { } @@ -188,4 +189,13 @@ public IntWritable evaluate(TimestampWritable i) { } } + public IntWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + intWritable.set(i.getBigDecimal().intValue()); + return intWritable; + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index c7ea66d..426bc64 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToLong extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); public UDFToLong() { } @@ -192,4 +193,13 @@ public LongWritable evaluate(TimestampWritable i) { } } + public LongWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + longWritable.set(i.getBigDecimal().longValue()); + return longWritable; + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index 558d405..5f42865 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -181,4 +182,13 @@ public ShortWritable evaluate(TimestampWritable i) { } } + public ShortWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + shortWritable.set(i.getBigDecimal().shortValue()); + return shortWritable; + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java index 4a38f8c..1d06eb3 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -141,10 +142,19 @@ public Text evaluate(TimestampWritable i) { } } + public Text evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + t.set(i.toString()); + return t; + } + } + public Text evaluate (BytesWritable bw) { if (null == bw) { return null; -} + } t.set(bw.getBytes(),0,bw.getLength()); return t; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java index 43ee547..d5c8e14 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java @@ -102,6 +102,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -110,6 +111,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCorrelationEvaluator(); case STRING: case BOOLEAN: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java index fdcedfb..f7f24f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java @@ -93,6 +93,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -101,6 +102,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceEvaluator(); case STRING: case BOOLEAN: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java index ef3023e..ecf7151 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java @@ -67,6 +67,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -75,6 +76,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceSampleEvaluator(); case STRING: case BOOLEAN: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java index e0f81e0..b31f1da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java @@ -45,7 +45,7 @@ /** * Computes an approximate histogram of a numerical column using a user-specified number of bins. - * + * * The output is an array of (x,y) pairs as Hive struct objects that represents the histogram's * bin centers and heights. */ @@ -72,7 +72,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE throw new UDFArgumentTypeException(parameters.length - 1, "Please specify exactly two arguments."); } - + // validate the first parameter, which is the expression to compute over if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, @@ -87,6 +87,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; case STRING: case BOOLEAN: @@ -170,7 +171,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc @Override public Object terminatePartial(AggregationBuffer agg) throws HiveException { - // Return a single ArrayList where the first element is the number of histogram bins, + // Return a single ArrayList where the first element is the number of histogram bins, // and subsequent elements represent histogram (x,y) pairs. StdAgg myagg = (StdAgg) agg; return myagg.histogram.serialize(); @@ -233,7 +234,7 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep } - // Aggregation buffer definition and manipulation methods + // Aggregation buffer definition and manipulation methods static class StdAgg implements AggregationBuffer { NumericHistogram histogram; // the histogram object }; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java index 4193a97..eab6e1e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java @@ -46,7 +46,7 @@ /** * Computes an approximate percentile (quantile) from an approximate histogram, for very * large numbers of rows where the regular percentile() UDAF might run out of memory. - * + * * The input is a single double value or an array of double values representing the quantiles * requested. The output, corresponding to the input, is either an single double value or an * array of doubles that are the quantile values. @@ -59,7 +59,7 @@ extended = "'expr' can be any numeric column, including doubles and floats, and 'pc' is " + "either a single double/float with a requested percentile, or an array of double/" + "float with multiple percentiles. If 'nb' is not specified, the default " + - "approximation is done with 10,000 histogram bins, which means that if there are " + + "approximation is done with 10,000 histogram bins, which means that if there are " + "10,000 or fewer unique values in 'expr', you can expect an exact result. The " + "percentile() function always computes an exact percentile and can run out of " + "memory if there are too many unique values in a column, which necessitates " + @@ -77,7 +77,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S throw new UDFArgumentTypeException(parameters.length - 1, "Please specify either two or three arguments."); } - + // Validate the first parameter, which is the expression to compute over. This should be a // numeric primitive type. if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { @@ -93,6 +93,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; default: throw new UDFArgumentTypeException(0, @@ -147,7 +148,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[1])) { throw new UDFArgumentTypeException(1, - "The second argument must be a constant, but " + parameters[1].getTypeName() + + "The second argument must be a constant, but " + parameters[1].getTypeName() + " was passed instead."); } @@ -172,7 +173,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[2])) { throw new UDFArgumentTypeException(2, - "The third argument must be a constant, but " + parameters[2].getTypeName() + + "The third argument must be a constant, but " + parameters[2].getTypeName() + " was passed instead."); } } @@ -184,7 +185,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S return new GenericUDAFSinglePercentileApproxEvaluator(); } } - + public static class GenericUDAFSinglePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -234,7 +235,7 @@ public Object terminate(AggregationBuffer agg) throws HiveException { } } - + public static class GenericUDAFMultiplePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -299,7 +300,7 @@ public Object terminate(AggregationBuffer agg) throws HiveException { @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { - if(partial == null) { + if(partial == null) { return; } PercentileAggBuf myagg = (PercentileAggBuf) agg; @@ -316,7 +317,7 @@ public void merge(AggregationBuffer agg, Object partial) throws HiveException { } // merge histograms - myagg.histogram.merge(partialHistogram); + myagg.histogram.merge(partialHistogram); } @Override @@ -382,7 +383,7 @@ public AggregationBuffer getNewAggregationBuffer() throws HiveException { } else { result = new double[1]; result[0] = PrimitiveObjectInspectorUtils.getDouble( - quantileObj, + quantileObj, (PrimitiveObjectInspector)quantileOI); } for(int ii = 0; ii < result.length; ++ii) { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java index 2a1a617..5d0aa50 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java @@ -28,7 +28,7 @@ /** * Compute the standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "std,stddev,stddev_pop", value = "_FUNC_(x) - Returns the standard deviation of a set of numbers") @@ -56,6 +56,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdEvaluator(); case BOOLEAN: default: @@ -68,7 +69,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) /** * Compute the standard deviation by extending GenericUDAFVarianceEvaluator * and overriding the terminate() method of the evaluator. - * + * */ public static class GenericUDAFStdEvaluator extends GenericUDAFVarianceEvaluator { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java index d5791ed..cde947c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java @@ -28,7 +28,7 @@ /** * Compute the sample standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "stddev_samp", value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers") @@ -55,6 +55,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdSampleEvaluator(); case BOOLEAN: default: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 5a20f87..f80491d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.hive.ql.udf.generic; +import java.math.BigDecimal; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -66,6 +69,8 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DOUBLE: case STRING: return new GenericUDAFSumDouble(); + case DECIMAL: + return new GenericUDAFSumBigDecimal(); case BOOLEAN: default: throw new UDFArgumentTypeException(0, @@ -75,6 +80,89 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) } /** + * GenericUDAFSumBigDecimal. + * + */ + public static class GenericUDAFSumBigDecimal extends GenericUDAFEvaluator { + private PrimitiveObjectInspector inputOI; + private BigDecimalWritable result; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { + assert (parameters.length == 1); + super.init(m, parameters); + result = new BigDecimalWritable(BigDecimal.ZERO); + inputOI = (PrimitiveObjectInspector) parameters[0]; + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + /** class for storing double sum value. */ + static class SumBigDecimalAgg implements AggregationBuffer { + boolean empty; + BigDecimal sum; + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + SumBigDecimalAgg agg = new SumBigDecimalAgg(); + reset(agg); + return agg; + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg bdAgg = (SumBigDecimalAgg) agg; + bdAgg.empty = true; + bdAgg.sum = BigDecimal.ZERO; + } + + boolean warned = false; + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { + assert (parameters.length == 1); + try { + merge(agg, parameters[0]); + } catch (NumberFormatException e) { + if (!warned) { + warned = true; + LOG.warn(getClass().getSimpleName() + " " + + StringUtils.stringifyException(e)); + LOG + .warn(getClass().getSimpleName() + + " ignoring similar exceptions."); + } + } + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException { + return terminate(agg); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException { + if (partial != null) { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + myagg.empty = false; + myagg.sum = myagg.sum.add( + PrimitiveObjectInspectorUtils.getBigDecimal(partial, inputOI)); + } + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + if (myagg.empty) { + return null; + } + result.set(myagg.sum); + return result; + } + + } + + /** * GenericUDAFSumDouble. * */ diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java index 0b40d5c..7bba95c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java @@ -43,7 +43,7 @@ /** * Compute the variance. This class is extended by: GenericUDAFVarianceSample * GenericUDAFStd GenericUDAFStdSample - * + * */ @Description(name = "variance,var_pop", value = "_FUNC_(x) - Returns the variance of a set of numbers") @@ -72,6 +72,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceEvaluator(); case BOOLEAN: default: @@ -85,18 +86,18 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE * Evaluate the variance using the algorithm described by Chan, Golub, and LeVeque in * "Algorithms for computing the sample variance: analysis and recommendations" * The American Statistician, 37 (1983) pp. 242--247. - * + * * variance = variance1 + variance2 + n/(m*(m+n)) * pow(((m/n)*t1 - t2),2) - * + * * where: - variance is sum[x-avg^2] (this is actually n times the variance) * and is updated at every step. - n is the count of elements in chunk1 - m is - * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = + * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = * sum of elements in chunk2. * * This algorithm was proven to be numerically stable by J.L. Barlow in * "Error analysis of a pairwise summation algorithm to compute sample variance" * Numer. Math, 58 (1991) pp. 583--590 - * + * */ public static class GenericUDAFVarianceEvaluator extends GenericUDAFEvaluator { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java index 65d860d..fa549e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java @@ -28,7 +28,7 @@ /** * Compute the sample variance by extending GenericUDAFVariance and overriding * the terminate() method of the evaluator. - * + * */ @Description(name = "var_samp", value = "_FUNC_(x) - Returns the sample variance of a set of numbers") @@ -56,6 +56,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceSampleEvaluator(); case BOOLEAN: default: diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java new file mode 100644 index 0000000..d6776d1 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.BigDecimalConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +@Description(name = "decimal", value = "_FUNC_(a) - cast a to decimal") +public class GenericUDFToDecimal extends GenericUDF { + + private PrimitiveObjectInspector argumentOI; + private BigDecimalConverter bdConverter; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 1) { + throw new UDFArgumentLengthException( + "The function DECIMAL requires at least one argument, got " + + arguments.length); + } + try { + argumentOI = (PrimitiveObjectInspector) arguments[0]; + } catch (ClassCastException e) { + throw new UDFArgumentException( + "The function DECIMAL takes only primitive types"); + } + + bdConverter = new BigDecimalConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector); + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + if (o0 == null) { + return null; + } + + return bdConverter.convert(o0); + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + StringBuilder sb = new StringBuilder(); + sb.append("CAST( "); + sb.append(children[0]); + sb.append(" AS DECIMAL)"); + return sb.toString(); + } + +} diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java new file mode 100644 index 0000000..0db19ac --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.util.List; +import java.util.LinkedList; +import java.lang.reflect.Method; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; + +public class TestFunctionRegistry extends TestCase { + + public class TestUDF { + public void same(DoubleWritable x, DoubleWritable y) {} + public void same(BigDecimalWritable x, BigDecimalWritable y) {} + public void one(IntWritable x, BigDecimalWritable y) {} + public void one(IntWritable x, DoubleWritable y) {} + public void one(IntWritable x, IntWritable y) {} + public void mismatch(TimestampWritable x, BigDecimalWritable y) {} + public void mismatch(BytesWritable x, DoubleWritable y) {} + } + + @Override + protected void setUp() { + } + + private void implicit(TypeInfo a, TypeInfo b, boolean convertible) { + assertEquals(convertible, FunctionRegistry.implicitConvertable(a,b)); + } + + public void testImplicitConversion() { + implicit(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, true); + implicit(TypeInfoFactory.floatTypeInfo, TypeInfoFactory.decimalTypeInfo, true); + implicit(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, true); + implicit(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, true); + implicit(TypeInfoFactory.timestampTypeInfo, TypeInfoFactory.decimalTypeInfo, false); + } + + private void verify(Class udf, String name, TypeInfo ta, TypeInfo tb, + Class a, Class b, boolean throwException) { + List args = new LinkedList(); + args.add(ta); + args.add(tb); + + Method result = null; + + try { + result = FunctionRegistry.getMethodInternal(udf, name, false, args); + } catch (UDFArgumentException e) { + assert(throwException); + return; + } + assert(!throwException); + assertEquals(2, result.getParameterTypes().length); + assertEquals(result.getParameterTypes()[0], a); + assertEquals(result.getParameterTypes()[1], b); + } + + public void testGetMethodInternal() { + + verify(TestUDF.class, "same", TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, + DoubleWritable.class, DoubleWritable.class, false); + + verify(TestUDF.class, "same", TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.doubleTypeInfo, + DoubleWritable.class, DoubleWritable.class, false); + + verify(TestUDF.class, "same", TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + BigDecimalWritable.class, BigDecimalWritable.class, false); + + verify(TestUDF.class, "same", TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.doubleTypeInfo, + BigDecimalWritable.class, BigDecimalWritable.class, false); + + verify(TestUDF.class, "same", TypeInfoFactory.decimalTypeInfo, TypeInfoFactory.decimalTypeInfo, + BigDecimalWritable.class, BigDecimalWritable.class, false); + + verify(TestUDF.class, "one", TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + IntWritable.class, BigDecimalWritable.class, false); + + verify(TestUDF.class, "one", TypeInfoFactory.intTypeInfo, TypeInfoFactory.floatTypeInfo, + IntWritable.class, DoubleWritable.class, false); + + verify(TestUDF.class, "one", TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, + IntWritable.class, IntWritable.class, false); + + verify(TestUDF.class, "mismatch", TypeInfoFactory.voidTypeInfo, TypeInfoFactory.intTypeInfo, + null, null, true); + } + + private void common(TypeInfo a, TypeInfo b, TypeInfo result) { + assertEquals(FunctionRegistry.getCommonClass(a,b), result); + } + + public void testCommonClass() { + common(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + common(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.stringTypeInfo); + common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + common(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.stringTypeInfo); + } + + private void comparison(TypeInfo a, TypeInfo b, TypeInfo result) { + assertEquals(FunctionRegistry.getCommonClassForComparison(a,b), result); + } + + public void testCommonClassComparison() { + comparison(TypeInfoFactory.intTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + comparison(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.decimalTypeInfo, + TypeInfoFactory.decimalTypeInfo); + comparison(TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.stringTypeInfo, + TypeInfoFactory.doubleTypeInfo); + } + + @Override + protected void tearDown() { + } +} diff --git ql/src/test/queries/clientpositive/decimal_1.q ql/src/test/queries/clientpositive/decimal_1.q new file mode 100644 index 0000000..063afa8 --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_1.q @@ -0,0 +1,17 @@ +drop table decimal_1; + +create table decimal_1 (t decimal); +alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + +insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1; +select cast(t as boolean) from decimal_1 limit 1; +select cast(t as tinyint) from decimal_1 limit 1; +select cast(t as smallint) from decimal_1 limit 1; +select cast(t as int) from decimal_1 limit 1; +select cast(t as bigint) from decimal_1 limit 1; +select cast(t as float) from decimal_1 limit 1; +select cast(t as double) from decimal_1 limit 1; +select cast(t as string) from decimal_1 limit 1; + +drop table decimal_1; diff --git ql/src/test/queries/clientpositive/decimal_2.q ql/src/test/queries/clientpositive/decimal_2.q new file mode 100644 index 0000000..1e3ad3e --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_2.q @@ -0,0 +1,35 @@ +drop table decimal_2; + +create table decimal_2 (t decimal); +alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; + +insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1; + +select cast(t as boolean) from decimal_2 limit 1; +select cast(t as tinyint) from decimal_2 limit 1; +select cast(t as smallint) from decimal_2 limit 1; +select cast(t as int) from decimal_2 limit 1; +select cast(t as bigint) from decimal_2 limit 1; +select cast(t as float) from decimal_2 limit 1; +select cast(t as double) from decimal_2 limit 1; +select cast(t as string) from decimal_2 limit 1; + +insert overwrite table decimal_2 + select cast('3404045.5044003' as decimal) from src limit 1; + +select cast(t as boolean) from decimal_2 limit 1; +select cast(t as tinyint) from decimal_2 limit 1; +select cast(t as smallint) from decimal_2 limit 1; +select cast(t as int) from decimal_2 limit 1; +select cast(t as bigint) from decimal_2 limit 1; +select cast(t as float) from decimal_2 limit 1; +select cast(t as double) from decimal_2 limit 1; +select cast(t as string) from decimal_2 limit 1; + +select cast(3.14 as decimal) from decimal_2 limit 1; +select cast(cast(3.14 as float) as decimal) from decimal_2 limit 1; +select cast(cast('2012-12-19 11:12:19.1234567' as timestamp) as decimal) from decimal_2 limit 1; +select cast(0.99999999999999999999 as decimal) from decimal_2 limit 1; +select cast('0.99999999999999999999' as decimal) from decimal_2 limit 1; +drop table decimal_2; diff --git ql/src/test/queries/clientpositive/decimal_3.q ql/src/test/queries/clientpositive/decimal_3.q new file mode 100644 index 0000000..2fed7ad --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_3.q @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS DECIMAL_3; + +CREATE TABLE DECIMAL_3(key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_3; + +SELECT * FROM DECIMAL_3; + +SELECT * FROM DECIMAL_3 ORDER BY key; + +SELECT * FROM DECIMAL_3 ORDER BY key DESC; + +SELECT * FROM DECIMAL_3 ORDER BY (key, value); + +SELECT DISTINCT key FROM DECIMAL_3; + +SELECT key, sum(value) FROM DECIMAL_3 GROUP BY key ORDER BY key; + +SELECT value, sum(key) FROM DECIMAL_3 GROUP BY value; + +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key); + +SELECT * FROM DECIMAL_3 WHERE key=3.14; + +SELECT * FROM DECIMAL_3 WHERE key=3.140; + +DROP TABLE DECIMAL_3; diff --git ql/src/test/queries/clientpositive/decimal_serde.q ql/src/test/queries/clientpositive/decimal_serde.q new file mode 100644 index 0000000..d199c85 --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_serde.q @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS DECIMAL_TEXT; +DROP TABLE IF EXISTS DECIMAL_RC; +DROP TABLE IF EXISTS DECIMAL_LAZY_COL; +DROP TABLE IF EXISTS DECIMAL_SEQUENCE; + +CREATE TABLE DECIMAL_TEXT (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_TEXT; + +SELECT * FROM DECIMAL_TEXT ORDER BY key; + +CREATE TABLE DECIMAL_RC +STORED AS RCFile AS +SELECT * FROM DECIMAL_TEXT; + +CREATE TABLE DECIMAL_LAZY_COL +ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" +STORED AS RCFile AS +SELECT * FROM DECIMAL_RC; + +CREATE TABLE DECIMAL_SEQUENCE +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\001' +COLLECTION ITEMS TERMINATED BY '\002' +MAP KEYS TERMINATED BY '\003' +STORED AS SEQUENCEFILE AS +SELECT * FROM DECIMAL_LAZY_COL ORDER BY key; + +SELECT * FROM DECIMAL_SEQUENCE; + +DROP TABLE IF EXISTS DECIMAL_TEXT; +DROP TABLE IF EXISTS DECIMAL_RC; +DROP TABLE IF EXISTS DECIMAL_LAZY_COL; +DROP TABLE IF EXISTS DECIMAL_SEQUENCE; diff --git ql/src/test/queries/clientpositive/decimal_udf.q ql/src/test/queries/clientpositive/decimal_udf.q new file mode 100644 index 0000000..59d2d17 --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_udf.q @@ -0,0 +1,112 @@ +DROP TABLE IF EXISTS DECIMAL_UDF; + +CREATE TABLE DECIMAL_UDF (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_UDF; + +-- addition +EXPLAIN SELECT key + key FROM DECIMAL_UDF; +SELECT key + key FROM DECIMAL_UDF; + +EXPLAIN SELECT key + value FROM DECIMAL_UDF; +SELECT key + value FROM DECIMAL_UDF; + +EXPLAIN SELECT key + (value/2) FROM DECIMAL_UDF; +SELECT key + (value/2) FROM DECIMAL_UDF; + +EXPLAIN SELECT key + '1.0' FROM DECIMAL_UDF; +SELECT key + '1.0' FROM DECIMAL_UDF; + +-- substraction +EXPLAIN SELECT key - key FROM DECIMAL_UDF; +SELECT key - key FROM DECIMAL_UDF; + +EXPLAIN SELECT key - value FROM DECIMAL_UDF; +SELECT key - value FROM DECIMAL_UDF; + +EXPLAIN SELECT key - (value/2) FROM DECIMAL_UDF; +SELECT key - (value/2) FROM DECIMAL_UDF; + +EXPLAIN SELECT key - '1.0' FROM DECIMAL_UDF; +SELECT key - '1.0' FROM DECIMAL_UDF; + +-- multiplication +EXPLAIN SELECT key * key FROM DECIMAL_UDF; +SELECT key * key FROM DECIMAL_UDF; + +EXPLAIN SELECT key * value FROM DECIMAL_UDF; +SELECT key * value FROM DECIMAL_UDF; + +EXPLAIN SELECT key * (value/2) FROM DECIMAL_UDF; +SELECT key * (value/2) FROM DECIMAL_UDF; + +EXPLAIN SELECT key * '2.0' FROM DECIMAL_UDF; +SELECT key * '2.0' FROM DECIMAL_UDF; + +-- division +EXPLAIN SELECT key / 0 FROM DECIMAL_UDF limit 1; +SELECT key / 0 FROM DECIMAL_UDF limit 1; + +EXPLAIN SELECT key / NULL FROM DECIMAL_UDF limit 1; +SELECT key / NULL FROM DECIMAL_UDF limit 1; + +EXPLAIN SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0; +SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0; + +EXPLAIN SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0; +SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0; + +EXPLAIN SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0; +SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0; + +EXPLAIN SELECT key / '2.0' FROM DECIMAL_UDF; +SELECT key / '2.0' FROM DECIMAL_UDF; + +-- abs +EXPLAIN SELECT abs(key) FROM DECIMAL_UDF; +SELECT abs(key) FROM DECIMAL_UDF; + +-- negative +EXPLAIN SELECT -key FROM DECIMAL_UDF; +SELECT -key FROM DECIMAL_UDF; + +-- positive +EXPLAIN SELECT +key FROM DECIMAL_UDF; +SELECT +key FROM DECIMAL_UDF; + +-- ceiling +EXPlAIN SELECT CEIL(key) FROM DECIMAL_UDF; +SELECT CEIL(key) FROM DECIMAL_UDF; + +-- floor +EXPLAIN SELECT FLOOR(key) FROM DECIMAL_UDF; +SELECT FLOOR(key) FROM DECIMAL_UDF; + +-- round +EXPLAIN SELECT ROUND(key, 2) FROM DECIMAL_UDF; +SELECT ROUND(key, 2) FROM DECIMAL_UDF; + +-- power +EXPLAIN SELECT POWER(key, 2) FROM DECIMAL_UDF; +SELECT POWER(key, 2) FROM DECIMAL_UDF; + +-- modulo +EXPLAIN SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF; +SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF; + +-- stddev, var +EXPLAIN SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value; +SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value; + +-- stddev_samp, var_samp +EXPLAIN SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value; +SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value; + +-- histogram +EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF; +SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF; + +DROP TABLE IF EXISTS DECIMAL_UDF; diff --git ql/src/test/queries/clientpositive/udf7.q ql/src/test/queries/clientpositive/udf7.q index b9580b7..d12394e 100644 --- ql/src/test/queries/clientpositive/udf7.q +++ ql/src/test/queries/clientpositive/udf7.q @@ -8,11 +8,15 @@ SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1; + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1; SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1), ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1; + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1; diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out index a3686d9..fdd013a 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_1.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToInteger with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToInteger with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_2.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_2.q.out index 494b22c..f23283b 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_2.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_2.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToByte with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToByte with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_3.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_3.q.out index 2428f3c..3cf06b3 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_3.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_3.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToShort with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToShort with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_4.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_4.q.out index 25ec117..3789cc2 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_4.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_4.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToLong with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToLong with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_5.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_5.q.out index 6152e47..55a776d 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_5.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_5.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToFloat with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToFloat with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/invalid_cast_from_binary_6.q.out ql/src/test/results/clientnegative/invalid_cast_from_binary_6.q.out index 6eff980..afd92e7 100644 --- ql/src/test/results/clientnegative/invalid_cast_from_binary_6.q.out +++ ql/src/test/results/clientnegative/invalid_cast_from_binary_6.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: create table tbl (a binary) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tbl -FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToDouble with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(string) _FUNC_(timestamp) +FAILED: SemanticException Line 0:-1 Wrong arguments 'a': No matching method for class org.apache.hadoop.hive.ql.udf.UDFToDouble with (binary). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(float) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientnegative/wrong_column_type.q.out ql/src/test/results/clientnegative/wrong_column_type.q.out index ab33949..d5dd66b 100644 --- ql/src/test/results/clientnegative/wrong_column_type.q.out +++ ql/src/test/results/clientnegative/wrong_column_type.q.out @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE dest1(a float) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@dest1 -FAILED: NoMatchingMethodException No matching method for class org.apache.hadoop.hive.ql.udf.UDFToFloat with (array). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) +FAILED: NoMatchingMethodException No matching method for class org.apache.hadoop.hive.ql.udf.UDFToFloat with (array). Possible choices: _FUNC_(void) _FUNC_(boolean) _FUNC_(tinyint) _FUNC_(smallint) _FUNC_(int) _FUNC_(bigint) _FUNC_(double) _FUNC_(string) _FUNC_(timestamp) _FUNC_(decimal) diff --git ql/src/test/results/clientpositive/decimal_1.q.out ql/src/test/results/clientpositive/decimal_1.q.out new file mode 100644 index 0000000..4cf6e0a --- /dev/null +++ ql/src/test/results/clientpositive/decimal_1.q.out @@ -0,0 +1,117 @@ +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_1 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_1 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] diff --git ql/src/test/results/clientpositive/decimal_2.q.out ql/src/test/results/clientpositive/decimal_2.q.out new file mode 100644 index 0000000..60dfa0a --- /dev/null +++ ql/src/test/results/clientpositive/decimal_2.q.out @@ -0,0 +1,273 @@ +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_2 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_2 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: insert overwrite table decimal_2 + select cast('3404045.5044003' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: insert overwrite table decimal_2 + select cast('3404045.5044003' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +13 +PREHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +-3827 +PREHOOK: query: select cast(t as int) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3404045 +PREHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3404045 +PREHOOK: query: select cast(t as float) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3404045.5 +PREHOOK: query: select cast(t as double) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3404045.5044003 +PREHOOK: query: select cast(t as string) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3404045.5044003 +PREHOOK: query: select cast(3.14 as decimal) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(3.14 as decimal) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3.14 +PREHOOK: query: select cast(cast(3.14 as float) as decimal) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(cast(3.14 as float) as decimal) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +3.14 +PREHOOK: query: select cast(cast('2012-12-19 11:12:19.1234567' as timestamp) as decimal) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(cast('2012-12-19 11:12:19.1234567' as timestamp) as decimal) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +1355944339.1234567 +PREHOOK: query: select cast(0.99999999999999999999 as decimal) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(0.99999999999999999999 as decimal) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +1 +PREHOOK: query: select cast('0.99999999999999999999' as decimal) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast('0.99999999999999999999' as decimal) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +0.99999999999999999999 +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] diff --git ql/src/test/results/clientpositive/decimal_3.q.out ql/src/test/results/clientpositive/decimal_3.q.out new file mode 100644 index 0000000..ad6190a --- /dev/null +++ ql/src/test/results/clientpositive/decimal_3.q.out @@ -0,0 +1,406 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_3(key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE DECIMAL_3(key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@DECIMAL_3 +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_3 +PREHOOK: type: LOAD +PREHOOK: Output: default@decimal_3 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_3 +POSTHOOK: type: LOAD +POSTHOOK: Output: default@decimal_3 +PREHOOK: query: SELECT * FROM DECIMAL_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-4.4E+3 4400 +1E+99 0 +1E-99 0 +0 0 +1E+2 100 +1E+1 10 +1 1 +0.1 0 +0.01 0 +2E+2 200 +2E+1 20 +2 2 +0 0 +0.2 0 +0.02 0 +0.3 0 +0.33 0 +0.333 0 +-0.3 0 +-0.33 0 +-0.333 0 +1 1 +2 2 +3.14 3 +-1.12 -1 +-1.12 -1 +-1.122 -11 +1.12 1 +1.122 1 +124 124 +125.2 125 +-1255.49 -1255 +3.14 3 +3.14 3 +3.14 4 +0.9999999999999999999999999 1 +-1234567890.123456789 -1234567890 +1234567890.12345678 1234567890 +PREHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 +-4.4E+3 4400 +-1255.49 -1255 +-1.122 -11 +-1.12 -1 +-1.12 -1 +-0.333 0 +-0.33 0 +-0.3 0 +0 0 +0 0 +1E-99 0 +0.01 0 +0.02 0 +0.1 0 +0.2 0 +0.3 0 +0.33 0 +0.333 0 +0.9999999999999999999999999 1 +1 1 +1 1 +1.12 1 +1.122 1 +2 2 +2 2 +3.14 3 +3.14 3 +3.14 3 +3.14 4 +1E+1 10 +2E+1 20 +1E+2 100 +124 124 +125.2 125 +2E+2 200 +1234567890.12345678 1234567890 +1E+99 0 +PREHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key DESC +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key DESC +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +1E+99 0 +1234567890.12345678 1234567890 +2E+2 200 +125.2 125 +124 124 +1E+2 100 +2E+1 20 +1E+1 10 +3.14 3 +3.14 3 +3.14 4 +3.14 3 +2 2 +2 2 +1.122 1 +1.12 1 +1 1 +1 1 +0.9999999999999999999999999 1 +0.333 0 +0.33 0 +0.3 0 +0.2 0 +0.1 0 +0.02 0 +0.01 0 +1E-99 0 +0 0 +0 0 +-0.3 0 +-0.33 0 +-0.333 0 +-1.12 -1 +-1.12 -1 +-1.122 -11 +-1255.49 -1255 +-4.4E+3 4400 +-1234567890.123456789 -1234567890 +PREHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY (key, value) +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY (key, value) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 +-4.4E+3 4400 +-1255.49 -1255 +-1.122 -11 +-1.12 -1 +-1.12 -1 +-0.333 0 +-0.33 0 +-0.3 0 +0 0 +0 0 +1E-99 0 +0.01 0 +0.02 0 +0.1 0 +0.2 0 +0.3 0 +0.33 0 +0.333 0 +0.9999999999999999999999999 1 +1 1 +1 1 +1.12 1 +1.122 1 +2 2 +2 2 +3.14 3 +3.14 3 +3.14 3 +3.14 4 +1E+1 10 +2E+1 20 +1E+2 100 +124 124 +125.2 125 +2E+2 200 +1234567890.12345678 1234567890 +1E+99 0 +PREHOOK: query: SELECT DISTINCT key FROM DECIMAL_3 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT DISTINCT key FROM DECIMAL_3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 +-4.4E+3 +-1255.49 +-1.122 +-1.12 +-0.333 +-0.33 +-0.3 +0.00 +1E-99 +0.01 +0.02 +0.1 +0.2 +0.3 +0.33 +0.333 +0.9999999999999999999999999 +1 +1.12 +1.122 +2 +3.14 +1E+1 +2E+1 +1E+2 +124 +125.2 +2E+2 +1234567890.12345678 +1E+99 +PREHOOK: query: SELECT key, sum(value) FROM DECIMAL_3 GROUP BY key ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, sum(value) FROM DECIMAL_3 GROUP BY key ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 +-4.4E+3 4400 +-1255.49 -1255 +-1.122 -11 +-1.12 -2 +-0.333 0 +-0.33 0 +-0.3 0 +0.00 0 +1E-99 0 +0.01 0 +0.02 0 +0.1 0 +0.2 0 +0.3 0 +0.33 0 +0.333 0 +0.9999999999999999999999999 1 +1 2 +1.12 1 +1.122 1 +2 4 +3.14 13 +1E+1 10 +2E+1 20 +1E+2 100 +124 124 +125.2 125 +2E+2 200 +1234567890.12345678 1234567890 +1E+99 0 +PREHOOK: query: SELECT value, sum(key) FROM DECIMAL_3 GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value, sum(key) FROM DECIMAL_3 GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890 -1234567890.123456789 +-1255 -1255.49 +-11 -1.122 +-1 -2.24 +0 1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 +1 5.2419999999999999999999999 +2 4 +3 9.42 +4 3.14 +10 1E+1 +20 2E+1 +100 1E+2 +124 124 +125 125.2 +200 2E+2 +4400 -4.4E+3 +1234567890 1234567890.12345678 +PREHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 -1234567890.123456789 -1234567890 +-4.4E+3 4400 -4.4E+3 4400 +-1255.49 -1255 -1255.49 -1255 +-1.122 -11 -1.122 -11 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-0.333 0 -0.333 0 +-0.33 0 -0.33 0 +-0.3 0 -0.3 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +1E-99 0 1E-99 0 +0.01 0 0.01 0 +0.02 0 0.02 0 +0.1 0 0.1 0 +0.2 0 0.2 0 +0.3 0 0.3 0 +0.33 0 0.33 0 +0.333 0 0.333 0 +0.9999999999999999999999999 1 0.9999999999999999999999999 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1.12 1 1.12 1 +1.122 1 1.122 1 +2 2 2 2 +2 2 2 2 +2 2 2 2 +2 2 2 2 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 4 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 4 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 4 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 4 +1E+1 10 1E+1 10 +2E+1 20 2E+1 20 +1E+2 100 1E+2 100 +124 124 124 124 +125.2 125 125.2 125 +2E+2 200 2E+2 200 +1234567890.12345678 1234567890 1234567890.12345678 1234567890 +1E+99 0 1E+99 0 +PREHOOK: query: SELECT * FROM DECIMAL_3 WHERE key=3.14 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 WHERE key=3.14 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +3.14 3 +3.14 3 +3.14 3 +3.14 4 +PREHOOK: query: SELECT * FROM DECIMAL_3 WHERE key=3.140 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 WHERE key=3.140 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +3.14 3 +3.14 3 +3.14 3 +3.14 4 +PREHOOK: query: DROP TABLE DECIMAL_3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_3 +PREHOOK: Output: default@decimal_3 +POSTHOOK: query: DROP TABLE DECIMAL_3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_3 +POSTHOOK: Output: default@decimal_3 diff --git ql/src/test/results/clientpositive/decimal_serde.q.out ql/src/test/results/clientpositive/decimal_serde.q.out new file mode 100644 index 0000000..10e9fd5 --- /dev/null +++ ql/src/test/results/clientpositive/decimal_serde.q.out @@ -0,0 +1,200 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_TEXT +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_TEXT +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_RC +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_RC +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_LAZY_COL +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_LAZY_COL +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_SEQUENCE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_SEQUENCE +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_TEXT (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE DECIMAL_TEXT (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@DECIMAL_TEXT +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_TEXT +PREHOOK: type: LOAD +PREHOOK: Output: default@decimal_text +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_TEXT +POSTHOOK: type: LOAD +POSTHOOK: Output: default@decimal_text +PREHOOK: query: SELECT * FROM DECIMAL_TEXT ORDER BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_text +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_TEXT ORDER BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_text +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 +-4.4E+3 4400 +-1255.49 -1255 +-1.122 -11 +-1.12 -1 +-1.12 -1 +-0.333 0 +-0.33 0 +-0.3 0 +0 0 +0 0 +1E-99 0 +0.01 0 +0.02 0 +0.1 0 +0.2 0 +0.3 0 +0.33 0 +0.333 0 +0.9999999999999999999999999 1 +1 1 +1 1 +1.12 1 +1.122 1 +2 2 +2 2 +3.14 3 +3.14 3 +3.14 3 +3.14 4 +1E+1 10 +2E+1 20 +1E+2 100 +124 124 +125.2 125 +2E+2 200 +1234567890.12345678 1234567890 +1E+99 0 +PREHOOK: query: CREATE TABLE DECIMAL_RC +STORED AS RCFile AS +SELECT * FROM DECIMAL_TEXT +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@decimal_text +POSTHOOK: query: CREATE TABLE DECIMAL_RC +STORED AS RCFile AS +SELECT * FROM DECIMAL_TEXT +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@decimal_text +POSTHOOK: Output: default@DECIMAL_RC +PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL +ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" +STORED AS RCFile AS +SELECT * FROM DECIMAL_RC +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@decimal_rc +POSTHOOK: query: CREATE TABLE DECIMAL_LAZY_COL +ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" +STORED AS RCFile AS +SELECT * FROM DECIMAL_RC +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@decimal_rc +POSTHOOK: Output: default@DECIMAL_LAZY_COL +PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\001' +COLLECTION ITEMS TERMINATED BY '\002' +MAP KEYS TERMINATED BY '\003' +STORED AS SEQUENCEFILE AS +SELECT * FROM DECIMAL_LAZY_COL ORDER BY key +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@decimal_lazy_col +POSTHOOK: query: CREATE TABLE DECIMAL_SEQUENCE +ROW FORMAT DELIMITED +FIELDS TERMINATED BY '\001' +COLLECTION ITEMS TERMINATED BY '\002' +MAP KEYS TERMINATED BY '\003' +STORED AS SEQUENCEFILE AS +SELECT * FROM DECIMAL_LAZY_COL ORDER BY key +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@decimal_lazy_col +POSTHOOK: Output: default@DECIMAL_SEQUENCE +PREHOOK: query: SELECT * FROM DECIMAL_SEQUENCE +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_sequence +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_SEQUENCE +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_sequence +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 +-4.4E+3 4400 +-1255.49 -1255 +-1.122 -11 +-1.12 -1 +-1.12 -1 +-0.333 0 +-0.33 0 +-0.3 0 +0 0 +0 0 +1E-99 0 +0.01 0 +0.02 0 +0.1 0 +0.2 0 +0.3 0 +0.33 0 +0.333 0 +0.9999999999999999999999999 1 +1 1 +1 1 +1.12 1 +1.122 1 +2 2 +2 2 +3.14 3 +3.14 3 +3.14 3 +3.14 4 +1E+1 10 +2E+1 20 +1E+2 100 +124 124 +125.2 125 +2E+2 200 +1234567890.12345678 1234567890 +1E+99 0 +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_TEXT +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_text +PREHOOK: Output: default@decimal_text +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_TEXT +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_text +POSTHOOK: Output: default@decimal_text +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_RC +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_rc +PREHOOK: Output: default@decimal_rc +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_RC +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_rc +POSTHOOK: Output: default@decimal_rc +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_LAZY_COL +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_lazy_col +PREHOOK: Output: default@decimal_lazy_col +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_LAZY_COL +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_lazy_col +POSTHOOK: Output: default@decimal_lazy_col +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_SEQUENCE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_sequence +PREHOOK: Output: default@decimal_sequence +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_SEQUENCE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_sequence +POSTHOOK: Output: default@decimal_sequence diff --git ql/src/test/results/clientpositive/decimal_udf.q.out ql/src/test/results/clientpositive/decimal_udf.q.out new file mode 100644 index 0000000..be05146 --- /dev/null +++ ql/src/test/results/clientpositive/decimal_udf.q.out @@ -0,0 +1,2355 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_UDF (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE DECIMAL_UDF (key decimal, value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@DECIMAL_UDF +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_UDF +PREHOOK: type: LOAD +PREHOOK: Output: default@decimal_udf +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/kv7.txt' INTO TABLE DECIMAL_UDF +POSTHOOK: type: LOAD +POSTHOOK: Output: default@decimal_udf +PREHOOK: query: -- addition +EXPLAIN SELECT key + key FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- addition +EXPLAIN SELECT key + key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key + key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key + key FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key + key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-8.8E+3 +2E+99 +2E-99 +0 +2E+2 +2E+1 +2 +0.2 +0.02 +4E+2 +4E+1 +4 +0 +0.4 +0.04 +0.6 +0.66 +0.666 +-0.6 +-0.66 +-0.666 +2 +4 +6.28 +-2.24 +-2.24 +-2.244 +2.24 +2.244 +248 +250.4 +-2510.98 +6.28 +6.28 +6.28 +1.9999999999999999999999998 +-2469135780.246913578 +2469135780.24691356 +PREHOOK: query: EXPLAIN SELECT key + value FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key + value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key + value) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key + value FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key + value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +0E+2 +1E+99 +1E-99 +0 +2E+2 +2E+1 +2 +0.1 +0.01 +4E+2 +4E+1 +4 +0 +0.2 +0.02 +0.3 +0.33 +0.333 +-0.3 +-0.33 +-0.333 +2 +4 +6.14 +-2.12 +-2.12 +-12.122 +2.12 +2.122 +248 +250.2 +-2510.49 +6.14 +6.14 +7.14 +1.9999999999999999999999999 +-2469135780.123456789 +2469135780.12345678 +PREHOOK: query: EXPLAIN SELECT key + (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key + (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_TABLE_OR_COL key) (/ (TOK_TABLE_OR_COL value) 2)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key + (value / 2)) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key + (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key + (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-2.2E+3 +1E+99 +1E-99 +0.0 +1.5E+2 +15 +1.5 +0.1 +0.01 +3E+2 +3E+1 +3 +0.0 +0.2 +0.02 +0.3 +0.33 +0.333 +-0.3 +-0.33 +-0.333 +1.5 +3 +4.64 +-1.62 +-1.62 +-6.622 +1.62 +1.622 +186 +187.7 +-1882.99 +4.64 +4.64 +5.14 +1.4999999999999999999999999 +-1851851835.123456789 +1851851835.12345678 +PREHOOK: query: EXPLAIN SELECT key + '1.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key + '1.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_TABLE_OR_COL key) '1.0'))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key + '1.0') + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key + '1.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key + '1.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4399 +1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 +1.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 +1 +101 +11 +2 +1.1 +1.01 +201 +21 +3 +1 +1.2 +1.02 +1.3 +1.33 +1.333 +0.7 +0.67 +0.667 +2 +3 +4.14 +-0.12 +-0.12 +-0.122 +2.12 +2.122 +125 +126.2 +-1254.49 +4.14 +4.14 +4.14 +1.9999999999999999999999999 +-1234567889.123456789 +1234567891.12345678 +PREHOOK: query: -- substraction +EXPLAIN SELECT key - key FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- substraction +EXPLAIN SELECT key - key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (- (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key - key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key - key FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key - key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +0E+2 +0E+99 +0E-99 +0 +0E+2 +0E+1 +0 +0.0 +0.00 +0E+2 +0E+1 +0 +0 +0.0 +0.00 +0.0 +0.00 +0.000 +0.0 +0.00 +0.000 +0 +0 +0.00 +0.00 +0.00 +0.000 +0.00 +0.000 +0 +0.0 +0.00 +0.00 +0.00 +0.00 +0E-25 +0E-9 +0E-8 +PREHOOK: query: EXPLAIN SELECT key - value FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key - value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (- (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key - value) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key - value FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key - value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-8.8E+3 +1E+99 +1E-99 +0 +0E+2 +0E+1 +0 +0.1 +0.01 +0E+2 +0E+1 +0 +0 +0.2 +0.02 +0.3 +0.33 +0.333 +-0.3 +-0.33 +-0.333 +0 +0 +0.14 +-0.12 +-0.12 +9.878 +0.12 +0.122 +0 +0.2 +-0.49 +0.14 +0.14 +-0.86 +-1E-25 +-0.123456789 +0.12345678 +PREHOOK: query: EXPLAIN SELECT key - (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key - (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (- (TOK_TABLE_OR_COL key) (/ (TOK_TABLE_OR_COL value) 2)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key - (value / 2)) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key - (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key - (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-6.6E+3 +1E+99 +1E-99 +0.0 +5E+1 +5 +0.5 +0.1 +0.01 +1E+2 +1E+1 +1 +0.0 +0.2 +0.02 +0.3 +0.33 +0.333 +-0.3 +-0.33 +-0.333 +0.5 +1 +1.64 +-0.62 +-0.62 +4.378 +0.62 +0.622 +62 +62.7 +-627.99 +1.64 +1.64 +1.14 +0.4999999999999999999999999 +-617283945.123456789 +617283945.12345678 +PREHOOK: query: EXPLAIN SELECT key - '1.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key - '1.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (- (TOK_TABLE_OR_COL key) '1.0'))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key - '1.0') + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key - '1.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key - '1.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4401 +999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +-0.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +-1 +99 +9 +0 +-0.9 +-0.99 +199 +19 +1 +-1 +-0.8 +-0.98 +-0.7 +-0.67 +-0.667 +-1.3 +-1.33 +-1.333 +0 +1 +2.14 +-2.12 +-2.12 +-2.122 +0.12 +0.122 +123 +124.2 +-1256.49 +2.14 +2.14 +2.14 +-1E-25 +-1234567891.123456789 +1234567889.12345678 +PREHOOK: query: -- multiplication +EXPLAIN SELECT key * key FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- multiplication +EXPLAIN SELECT key * key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (* (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key * key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key * key FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key * key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +1.936E+7 +1E+198 +1E-198 +0 +1E+4 +1E+2 +1 +0.01 +0.0001 +4E+4 +4E+2 +4 +0 +0.04 +0.0004 +0.09 +0.1089 +0.110889 +0.09 +0.1089 +0.110889 +1 +4 +9.8596 +1.2544 +1.2544 +1.258884 +1.2544 +1.258884 +15376 +15675.04 +1576255.1401 +9.8596 +9.8596 +9.8596 +0.99999999999999999999999980000000000000000000000001 +1524157875323883675.019051998750190521 +1524157875323883652.7968299765279684 +PREHOOK: query: EXPLAIN SELECT key * value FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key * value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (* (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key * value) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key * value FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key * value FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-1.936E+7 +0E+99 +0E-99 +0 +1E+4 +1E+2 +1 +0.0 +0.00 +4E+4 +4E+2 +4 +0 +0.0 +0.00 +0.0 +0.00 +0.000 +0.0 +0.00 +0.000 +1 +4 +9.42 +1.12 +1.12 +12.342 +1.12 +1.122 +15376 +1.565E+4 +1575639.95 +9.42 +9.42 +12.56 +0.9999999999999999999999999 +1524157875171467887.50190521 +1524157875171467876.3907942 +PREHOOK: query: EXPLAIN SELECT key * (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key * (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (* (TOK_TABLE_OR_COL key) (/ (TOK_TABLE_OR_COL value) 2)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key * (value / 2)) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key * (value/2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key * (value/2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-9.68E+6 +0E+98 +0E-100 +0.0 +5E+3 +5E+1 +0.5 +0.00 +0.000 +2E+4 +2E+2 +2 +0.0 +0.00 +0.000 +0.00 +0.000 +0.0000 +0.00 +0.000 +0.0000 +0.5 +2 +4.71 +0.56 +0.56 +6.171 +0.56 +0.561 +7688 +7825 +787819.975 +4.71 +4.71 +6.28 +0.49999999999999999999999995 +762078937585733943.750952605 +762078937585733938.1953971 +PREHOOK: query: EXPLAIN SELECT key * '2.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key * '2.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (* (TOK_TABLE_OR_COL key) '2.0'))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key * '2.0') + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key * '2.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key * '2.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-8.8E+3 +2E+99 +2E-99 +0 +2E+2 +2E+1 +2 +0.2 +0.02 +4E+2 +4E+1 +4 +0 +0.4 +0.04 +0.6 +0.66 +0.666 +-0.6 +-0.66 +-0.666 +2 +4 +6.28 +-2.24 +-2.24 +-2.244 +2.24 +2.244 +248 +250.4 +-2510.98 +6.28 +6.28 +6.28 +1.9999999999999999999999998 +-2469135780.246913578 +2469135780.24691356 +PREHOOK: query: -- division +EXPLAIN SELECT key / 0 FROM DECIMAL_UDF limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: -- division +EXPLAIN SELECT key / 0 FROM DECIMAL_UDF limit 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) 0))) (TOK_LIMIT 1))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key / 0) + type: decimal + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 1 + + +PREHOOK: query: SELECT key / 0 FROM DECIMAL_UDF limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / 0 FROM DECIMAL_UDF limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +NULL +PREHOOK: query: EXPLAIN SELECT key / NULL FROM DECIMAL_UDF limit 1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key / NULL FROM DECIMAL_UDF limit 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) TOK_NULL))) (TOK_LIMIT 1))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key / null) + type: decimal + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 1 + + +PREHOOK: query: SELECT key / NULL FROM DECIMAL_UDF limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / NULL FROM DECIMAL_UDF limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +NULL +PREHOOK: query: EXPLAIN SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL key)))) (TOK_WHERE (and (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL key)) (<> (TOK_TABLE_OR_COL key) 0))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Filter Operator + predicate: + expr: (key is not null and (key <> 0)) + type: boolean + Select Operator + expressions: + expr: (key / key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +PREHOOK: query: EXPLAIN SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) (TOK_WHERE (and (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value)) (<> (TOK_TABLE_OR_COL value) 0))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Filter Operator + predicate: + expr: (value is not null and (value <> 0)) + type: boolean + Select Operator + expressions: + expr: (key / value) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-1 +1 +1 +1 +1 +1 +1 +1 +1 +1.04666666666666666666666666666666666666666666666666666666666666667 +1.12 +1.12 +0.102 +1.12 +1.122 +1 +1.0016 +1.00039043824701195219123505976095617529880478087649402390438247012 +1.04666666666666666666666666666666666666666666666666666666666666667 +1.04666666666666666666666666666666666666666666666666666666666666667 +0.785 +0.9999999999999999999999999 +1.0000000001 +1.00000000009999999270999993366099939631509450646736000885297608056 +PREHOOK: query: EXPLAIN SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) (/ (TOK_TABLE_OR_COL value) 2)))) (TOK_WHERE (and (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL value)) (<> (TOK_TABLE_OR_COL value) 0))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Filter Operator + predicate: + expr: (value is not null and (value <> 0)) + type: boolean + Select Operator + expressions: + expr: (key / (value / 2)) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-2 +2 +2 +2 +2 +2 +2 +2 +2 +2.09333333333333333333333333333333333333333333333333333333333333333 +2.24 +2.24 +0.204 +2.24 +2.244 +2 +2.0032 +2.00078087649402390438247011952191235059760956175298804780876494024 +2.09333333333333333333333333333333333333333333333333333333333333333 +2.09333333333333333333333333333333333333333333333333333333333333333 +1.57 +1.9999999999999999999999998 +2.0000000002 +2.00000000019999998541999986732199879263018901293472001770595216112 +PREHOOK: query: EXPLAIN SELECT key / '2.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT key / '2.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (/ (TOK_TABLE_OR_COL key) '2.0'))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (key / '2.0') + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT key / '2.0' FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT key / '2.0' FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-2.2E+3 +5E+98 +0E-65 +0E-65 +5E+1 +5 +0.5 +0.05 +0.005 +1E+2 +1E+1 +1 +0E-65 +0.1 +0.01 +0.15 +0.165 +0.1665 +-0.15 +-0.165 +-0.1665 +0.5 +1 +1.57 +-0.56 +-0.56 +-0.561 +0.56 +0.561 +62 +62.6 +-627.745 +1.57 +1.57 +1.57 +0.49999999999999999999999995 +-617283945.0617283945 +617283945.06172839 +PREHOOK: query: -- abs +EXPLAIN SELECT abs(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- abs +EXPLAIN SELECT abs(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION abs (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: abs(key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT abs(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT abs(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +4.4E+3 +1E+99 +1E-99 +0 +1E+2 +1E+1 +1 +0.1 +0.01 +2E+2 +2E+1 +2 +0 +0.2 +0.02 +0.3 +0.33 +0.333 +0.3 +0.33 +0.333 +1 +2 +3.14 +1.12 +1.12 +1.122 +1.12 +1.122 +124 +125.2 +1255.49 +3.14 +3.14 +3.14 +0.9999999999999999999999999 +1234567890.123456789 +1234567890.12345678 +PREHOOK: query: -- negative +EXPLAIN SELECT -key FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- negative +EXPLAIN SELECT -key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (- (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: (- key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT -key FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT -key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +4.4E+3 +-1E+99 +-1E-99 +0 +-1E+2 +-1E+1 +-1 +-0.1 +-0.01 +-2E+2 +-2E+1 +-2 +0 +-0.2 +-0.02 +-0.3 +-0.33 +-0.333 +0.3 +0.33 +0.333 +-1 +-2 +-3.14 +1.12 +1.12 +1.122 +-1.12 +-1.122 +-124 +-125.2 +1255.49 +-3.14 +-3.14 +-3.14 +-0.9999999999999999999999999 +1234567890.123456789 +-1234567890.12345678 +PREHOOK: query: -- positive +EXPLAIN SELECT +key FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- positive +EXPLAIN SELECT +key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: key + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT +key FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT +key FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4.4E+3 +1E+99 +1E-99 +0 +1E+2 +1E+1 +1 +0.1 +0.01 +2E+2 +2E+1 +2 +0 +0.2 +0.02 +0.3 +0.33 +0.333 +-0.3 +-0.33 +-0.333 +1 +2 +3.14 +-1.12 +-1.12 +-1.122 +1.12 +1.122 +124 +125.2 +-1255.49 +3.14 +3.14 +3.14 +0.9999999999999999999999999 +-1234567890.123456789 +1234567890.12345678 +PREHOOK: query: -- ceiling +EXPlAIN SELECT CEIL(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- ceiling +EXPlAIN SELECT CEIL(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION CEIL (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: ceil(key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT CEIL(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT CEIL(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4.4E+3 +1E+99 +1 +0 +1E+2 +1E+1 +1 +1 +1 +2E+2 +2E+1 +2 +0 +1 +1 +1 +1 +1 +0.0 +0.00 +0.000 +1 +2 +4 +-1 +-1 +-1 +2 +2 +124 +126 +-1255 +4 +4 +4 +1 +-1.23456789E+9 +1234567891 +PREHOOK: query: -- floor +EXPLAIN SELECT FLOOR(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- floor +EXPLAIN SELECT FLOOR(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION FLOOR (TOK_TABLE_OR_COL key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: floor(key) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT FLOOR(key) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT FLOOR(key) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4.4E+3 +1E+99 +0E-99 +0 +1E+2 +1E+1 +1 +0.0 +0.00 +2E+2 +2E+1 +2 +0 +0.0 +0.00 +0.0 +0.00 +0.000 +-1 +-1 +-1 +1 +2 +3 +-2 +-2 +-2 +1 +1 +124 +125 +-1256 +3 +3 +3 +0E-25 +-1234567891 +1.23456789E+9 +PREHOOK: query: -- round +EXPLAIN SELECT ROUND(key, 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- round +EXPLAIN SELECT ROUND(key, 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_TABLE_OR_COL key) 2))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: round(key, 2) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT ROUND(key, 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT ROUND(key, 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-4.4E+3 +1E+99 +0E-99 +0 +1E+2 +1E+1 +1 +0.1 +0.01 +2E+2 +2E+1 +2 +0 +0.2 +0.02 +0.3 +0.33 +0.33 +-0.3 +-0.33 +-0.33 +1 +2 +3.14 +-1.12 +-1.12 +-1.12 +1.12 +1.12 +124 +125.2 +-1255.49 +3.14 +3.14 +3.14 +1 +-1234567890.12 +1234567890.12 +PREHOOK: query: -- power +EXPLAIN SELECT POWER(key, 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- power +EXPLAIN SELECT POWER(key, 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION POWER (TOK_TABLE_OR_COL key) 2))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: power(key, 2) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT POWER(key, 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT POWER(key, 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +1.936E+7 +1E+198 +1E-198 +0 +1E+4 +1E+2 +1 +0.01 +0.0001 +4E+4 +4E+2 +4 +0 +0.04 +0.0004 +0.09 +0.1089 +0.110889 +0.09 +0.1089 +0.110889 +1 +4 +9.8596 +1.2544 +1.2544 +1.258884 +1.2544 +1.258884 +15376 +15675.04 +1576255.1401 +9.8596 +9.8596 +9.8596 +0.99999999999999999999999980000000000000000000000001 +1524157875323883675.019051998750190521 +1524157875323883652.7968299765279684 +PREHOOK: query: -- modulo +EXPLAIN SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- modulo +EXPLAIN SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (% (+ (TOK_TABLE_OR_COL key) 1) (/ (TOK_TABLE_OR_COL key) 2)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: ((key + 1) % (key / 2)) + type: decimal + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-2199 +1 +NULL +NULL +1 +1 +0.0 +0.00 +0.000 +1 +1 +0 +NULL +0.0 +0.00 +0.1 +0.01 +0.001 +0.1 +0.01 +0.001 +0.0 +0 +1 +-0.12 +-0.12 +-0.122 +0.44 +0.439 +1 +1 +-626.745 +1 +1 +1 +1E-25 +-617283944.0617283945 +1 +PREHOOK: query: -- stddev, var +EXPLAIN SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value +PREHOOK: type: QUERY +POSTHOOK: query: -- stddev, var +EXPLAIN SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION stddev (TOK_TABLE_OR_COL key))) (TOK_SELEXPR (TOK_FUNCTION variance (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: value + type: int + expr: key + type: decimal + outputColumnNames: value, key + Group By Operator + aggregations: + expr: stddev(key) + expr: variance(key) + bucketGroup: false + keys: + expr: value + type: int + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: int + sort order: + + Map-reduce partition columns: + expr: _col0 + type: int + tag: -1 + value expressions: + expr: _col1 + type: struct + expr: _col2 + type: struct + Reduce Operator Tree: + Group By Operator + aggregations: + expr: stddev(VALUE._col0) + expr: variance(VALUE._col1) + bucketGroup: false + keys: + expr: KEY._col0 + type: int + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col1 + type: double + expr: _col2 + type: double + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-1234567890 0.0 0.0 +-1255 0.0 0.0 +-11 0.0 0.0 +-1 0.0 0.0 +0 2.5753937681885636E98 6.632653061224489E196 +1 0.05928102563215321 0.0035142400000000066 +2 0.0 0.0 +3 0.0 0.0 +4 0.0 0.0 +10 0.0 0.0 +20 0.0 0.0 +100 0.0 0.0 +124 0.0 0.0 +125 0.0 0.0 +200 0.0 0.0 +4400 0.0 0.0 +1234567890 0.0 0.0 +PREHOOK: query: -- stddev_samp, var_samp +EXPLAIN SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value +PREHOOK: type: QUERY +POSTHOOK: query: -- stddev_samp, var_samp +EXPLAIN SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION stddev_samp (TOK_TABLE_OR_COL key))) (TOK_SELEXPR (TOK_FUNCTION var_samp (TOK_TABLE_OR_COL key)))) (TOK_GROUPBY (TOK_TABLE_OR_COL value)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: value + type: int + expr: key + type: decimal + outputColumnNames: value, key + Group By Operator + aggregations: + expr: stddev_samp(key) + expr: var_samp(key) + bucketGroup: false + keys: + expr: value + type: int + mode: hash + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col0 + type: int + sort order: + + Map-reduce partition columns: + expr: _col0 + type: int + tag: -1 + value expressions: + expr: _col1 + type: struct + expr: _col2 + type: struct + Reduce Operator Tree: + Group By Operator + aggregations: + expr: stddev_samp(VALUE._col0) + expr: var_samp(VALUE._col1) + bucketGroup: false + keys: + expr: KEY._col0 + type: int + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: _col0 + type: int + expr: _col1 + type: double + expr: _col2 + type: double + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +-1234567890 0.0 0.0 +-1255 0.0 0.0 +-11 0.0 0.0 +-1 0.0 0.0 +0 2.6726124191242437E98 7.142857142857142E196 +1 0.06627820154470102 0.004392800000000008 +2 0.0 0.0 +3 0.0 0.0 +4 0.0 0.0 +10 0.0 0.0 +20 0.0 0.0 +100 0.0 0.0 +124 0.0 0.0 +125 0.0 0.0 +200 0.0 0.0 +4400 0.0 0.0 +1234567890 0.0 0.0 +PREHOOK: query: -- histogram +EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF +PREHOOK: type: QUERY +POSTHOOK: query: -- histogram +EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME DECIMAL_UDF))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION histogram_numeric (TOK_TABLE_OR_COL key) 3))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + decimal_udf + TableScan + alias: decimal_udf + Select Operator + expressions: + expr: key + type: decimal + outputColumnNames: key + Group By Operator + aggregations: + expr: histogram_numeric(key, 3) + bucketGroup: false + mode: hash + outputColumnNames: _col0 + Reduce Output Operator + sort order: + tag: -1 + value expressions: + expr: _col0 + type: array + Reduce Operator Tree: + Group By Operator + aggregations: + expr: histogram_numeric(VALUE._col0) + bucketGroup: false + mode: mergepartial + outputColumnNames: _col0 + Select Operator + expressions: + expr: _col0 + type: array> + outputColumnNames: _col0 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +POSTHOOK: query: SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_udf +#### A masked pattern was here #### +[{"x":-3.429369299009602E7,"y":36.0},{"x":1.2345678901234567E9,"y":1.0},{"x":1.0E99,"y":1.0}] +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_udf +PREHOOK: Output: default@decimal_udf +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_UDF +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_udf +POSTHOOK: Output: default@decimal_udf diff --git ql/src/test/results/clientpositive/udf7.q.out ql/src/test/results/clientpositive/udf7.q.out index 304ee3f..6b4eb11 100644 --- ql/src/test/results/clientpositive/udf7.q.out +++ ql/src/test/results/clientpositive/udf7.q.out @@ -18,7 +18,9 @@ SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1 + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), @@ -26,11 +28,13 @@ SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0), ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1 + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Lineage: dest1.c1 SIMPLE [] ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dest1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LN 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LN 0.0)) (TOK_SELEXPR (TOK_FUNCTION LN (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG2 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG10 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG10 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG10 (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG 2 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG 0.5 2)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.5)) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION EXP 2.0) 12)) (TOK_SELEXPR (TOK_FUNCTION POW 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 (- 3))) (TOK_SELEXPR (TOK_FUNCTION POWER 0.5 (- 3))) (TOK_SELEXPR (TOK_FUNCTION POWER 4 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 2))))) + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dest1))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LN 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LN 0.0)) (TOK_SELEXPR (TOK_FUNCTION LN (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG2 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG10 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG10 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG10 (- 1))) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION LOG 2 3.0) 12)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG 0.5 2)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.5)) (TOK_SELEXPR (TOK_FUNCTION ROUND (TOK_FUNCTION EXP 2.0) 12)) (TOK_SELEXPR (TOK_FUNCTION POW 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 (- 3))) (TOK_SELEXPR (TOK_FUNCTION POWER 0.5 (- 3))) (TOK_SELEXPR (TOK_FUNCTION POWER 4 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 2)) (TOK_SELEXPR (TOK_FUNCTION POWER (TOK_FUNCTION TOK_DECIMAL 1) (TOK_FUNCTION TOK_INT 0))) (TOK_SELEXPR (TOK_FUNCTION POWER (TOK_FUNCTION TOK_DECIMAL 2) (TOK_FUNCTION TOK_INT 3))) (TOK_SELEXPR (TOK_FUNCTION POW (TOK_FUNCTION TOK_DECIMAL 2) (TOK_FUNCTION TOK_INT 3)))))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -95,7 +99,13 @@ STAGE PLANS: type: double expr: power((- 1), 2) type: double - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 + expr: power(CAST( 1 AS DECIMAL), 0) + type: decimal + expr: power(CAST( 2 AS DECIMAL), 3) + type: decimal + expr: pow(CAST( 2 AS DECIMAL), 3) + type: decimal + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27 File Output Operator compressed: false GlobalTableId: 0 @@ -113,7 +123,9 @@ PREHOOK: query: SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), L ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1 + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### @@ -122,9 +134,11 @@ POSTHOOK: query: SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12), LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12), POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5), - POWER(-1, 0.5), POWER(-1, 2) FROM dest1 + POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)), + POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)), + POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: Lineage: dest1.c1 SIMPLE [] -1.098612288668 NULL NULL 1.098612288668 NULL NULL 1.584962500721 NULL NULL 0.47712125472 NULL NULL 1.584962500721 NULL NULL NULL -1.0 7.389056098931 8.0 8.0 0.125 8.0 2.0 NaN 1.0 +1.098612288668 NULL NULL 1.098612288668 NULL NULL 1.584962500721 NULL NULL 0.47712125472 NULL NULL 1.584962500721 NULL NULL NULL -1.0 7.389056098931 8.0 8.0 0.125 8.0 2.0 NaN 1.0 1 8 8 diff --git serde/if/serde.thrift serde/if/serde.thrift index e40c697..9847720 100644 --- serde/if/serde.thrift +++ serde/if/serde.thrift @@ -53,6 +53,7 @@ const string STRING_TYPE_NAME = "string"; const string DATE_TYPE_NAME = "date"; const string DATETIME_TYPE_NAME = "datetime"; const string TIMESTAMP_TYPE_NAME = "timestamp"; +const string DECIMAL_TYPE_NAME = "decimal"; const string BINARY_TYPE_NAME = "binary"; const string LIST_TYPE_NAME = "array"; @@ -63,7 +64,7 @@ const string UNION_TYPE_NAME = "uniontype"; const string LIST_COLUMNS = "columns"; const string LIST_COLUMN_TYPES = "columns.types"; -const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME BINARY_TYPE_NAME], +const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME], const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], diff --git serde/src/gen/thrift/gen-cpp/serde_constants.cpp serde/src/gen/thrift/gen-cpp/serde_constants.cpp index 6259225..3997026 100644 --- serde/src/gen/thrift/gen-cpp/serde_constants.cpp +++ serde/src/gen/thrift/gen-cpp/serde_constants.cpp @@ -63,6 +63,8 @@ serdeConstants::serdeConstants() { TIMESTAMP_TYPE_NAME = "timestamp"; + DECIMAL_TYPE_NAME = "decimal"; + BINARY_TYPE_NAME = "binary"; LIST_TYPE_NAME = "array"; @@ -89,6 +91,7 @@ serdeConstants::serdeConstants() { PrimitiveTypes.insert("date"); PrimitiveTypes.insert("datetime"); PrimitiveTypes.insert("timestamp"); + PrimitiveTypes.insert("decimal"); PrimitiveTypes.insert("binary"); CollectionTypes.insert("array"); diff --git serde/src/gen/thrift/gen-cpp/serde_constants.h serde/src/gen/thrift/gen-cpp/serde_constants.h index cb9c3e4..0a63308 100644 --- serde/src/gen/thrift/gen-cpp/serde_constants.h +++ serde/src/gen/thrift/gen-cpp/serde_constants.h @@ -41,6 +41,7 @@ class serdeConstants { std::string DATE_TYPE_NAME; std::string DATETIME_TYPE_NAME; std::string TIMESTAMP_TYPE_NAME; + std::string DECIMAL_TYPE_NAME; std::string BINARY_TYPE_NAME; std::string LIST_TYPE_NAME; std::string MAP_TYPE_NAME; diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java index db39e6e..28f8d6a 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -85,6 +85,8 @@ public static final String TIMESTAMP_TYPE_NAME = "timestamp"; + public static final String DECIMAL_TYPE_NAME = "decimal"; + public static final String BINARY_TYPE_NAME = "binary"; public static final String LIST_TYPE_NAME = "array"; @@ -113,6 +115,7 @@ PrimitiveTypes.add("date"); PrimitiveTypes.add("datetime"); PrimitiveTypes.add("timestamp"); + PrimitiveTypes.add("decimal"); PrimitiveTypes.add("binary"); } diff --git serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php index d95feef..130c17e 100644 --- serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php +++ serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php @@ -68,6 +68,8 @@ $GLOBALS['serde_CONSTANTS']['DATETIME_TYPE_NAME'] = "datetime"; $GLOBALS['serde_CONSTANTS']['TIMESTAMP_TYPE_NAME'] = "timestamp"; +$GLOBALS['serde_CONSTANTS']['DECIMAL_TYPE_NAME'] = "decimal"; + $GLOBALS['serde_CONSTANTS']['BINARY_TYPE_NAME'] = "binary"; $GLOBALS['serde_CONSTANTS']['LIST_TYPE_NAME'] = "array"; @@ -95,6 +97,7 @@ $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array( "date" => true, "datetime" => true, "timestamp" => true, + "decimal" => true, "binary" => true, ); diff --git serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py index 598db32..623bf0e 100644 --- serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py +++ serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py @@ -35,6 +35,7 @@ DATE_TYPE_NAME = "date" DATETIME_TYPE_NAME = "datetime" TIMESTAMP_TYPE_NAME = "timestamp" +DECIMAL_TYPE_NAME = "decimal" BINARY_TYPE_NAME = "binary" LIST_TYPE_NAME = "array" MAP_TYPE_NAME = "map" @@ -55,6 +56,7 @@ "date", "datetime", "timestamp", + "decimal", "binary", ]) CollectionTypes = set([ diff --git serde/src/gen/thrift/gen-rb/serde_constants.rb serde/src/gen/thrift/gen-rb/serde_constants.rb index bfe4a3b..bd17761 100644 --- serde/src/gen/thrift/gen-rb/serde_constants.rb +++ serde/src/gen/thrift/gen-rb/serde_constants.rb @@ -59,6 +59,8 @@ DATETIME_TYPE_NAME = %q"datetime" TIMESTAMP_TYPE_NAME = %q"timestamp" +DECIMAL_TYPE_NAME = %q"decimal" + BINARY_TYPE_NAME = %q"binary" LIST_TYPE_NAME = %q"array" @@ -86,6 +88,7 @@ PrimitiveTypes = Set.new([ %q"date", %q"datetime", %q"timestamp", + %q"decimal", %q"binary", ]) diff --git serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index e906a3f..4954b29 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -279,6 +280,10 @@ static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) { sb.append(txt.toString()); break; } + case DECIMAL: { + sb.append(((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + } default: throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index 450e063..a1cd4fe 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hive.serde2.binarysortable; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -33,6 +36,7 @@ import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -45,6 +49,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -106,6 +111,9 @@ StructObjectInspector rowObjectInspector; boolean[] columnSortOrderIsDesc; + + private static byte[] decimalBuffer = null; + private static Charset decimalCharSet = Charset.forName("US-ASCII"); @Override public void initialize(Configuration conf, Properties tbl) @@ -178,7 +186,7 @@ public Object deserialize(Writable blob) throws SerDeException { static Object deserialize(InputByteBuffer buffer, TypeInfo type, boolean invert, Object reuse) throws IOException { - + // Is this field a null? byte isNull = buffer.read(invert); if (isNull == 0) { @@ -370,6 +378,64 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } t.setBinarySortable(bytes, 0); return t; + + case DECIMAL: { + // See serialization of decimal for explanation (below) + + BigDecimalWritable bdw = (reuse == null ? new BigDecimalWritable() : + (BigDecimalWritable) reuse); + + int b = buffer.read(invert) - 1; + assert (b == 1 || b == -1 || b == 0); + boolean positive = b != -1; + + int factor = buffer.read(invert) ^ 0x80; + for (int i = 0; i < 3; i++) { + factor = (factor << 8) + (buffer.read(invert) & 0xff); + } + + if (!positive) { + factor = -factor; + } + + int start = buffer.tell(); + int length = 0; + + do { + b = buffer.read(positive ? invert : !invert); + assert(b != 1); + + if (b == 0) { + // end of digits + break; + } + + length++; + } while (true); + + if(decimalBuffer == null || decimalBuffer.length < length) { + decimalBuffer = new byte[length]; + } + + buffer.seek(start); + for (int i = 0; i < length; ++i) { + decimalBuffer[i] = buffer.read(positive ? invert : !invert); + } + + // read the null byte again + buffer.read(positive ? invert : !invert); + + String digits = new String(decimalBuffer, 0, length, decimalCharSet); + BigInteger bi = new BigInteger(digits); + BigDecimal bd = new BigDecimal(bi).scaleByPowerOfTen(factor-length); + + if (!positive) { + bd = bd.negate(); + } + + bdw.set(bd); + return bdw; + } default: { throw new RuntimeException("Unrecognized type: " @@ -377,6 +443,7 @@ static Object deserialize(InputByteBuffer buffer, TypeInfo type, } } } + case LIST: { ListTypeInfo ltype = (ListTypeInfo) type; TypeInfo etype = ltype.getListElementTypeInfo(); @@ -608,6 +675,47 @@ static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, } return; } + case DECIMAL: { + // decimals are encoded in three pieces: + // sign: 1, 2 or 3 for smaller, equal or larger than 0 respectively + // factor: Number that indicates the amount of digits you have to move + // the decimal point left or right until the resulting number is smaller + // than zero but has something other than 0 as the first digit. + // digits: which is a string of all the digits in the decimal. If the number + // is negative the binary string will be inverted to get the correct ordering. + // Example: 0.00123 + // Sign is 3 (bigger than 0) + // Factor is -2 (move decimal point 2 positions right) + // Digits are: 123 + + BigDecimalObjectInspector boi = (BigDecimalObjectInspector) poi; + BigDecimal dec = boi.getPrimitiveJavaObject(o).stripTrailingZeros(); + + // get the sign of the big decimal + int sign = dec.compareTo(BigDecimal.ZERO); + + // we'll encode the absolute value (sign is separate) + dec = dec.abs(); + + // get the scale factor to turn big decimal into a decimal < 1 + int factor = dec.precision() - dec.scale(); + factor = sign == 1 ? factor : -factor; + + // convert the absolute big decimal to string + dec.scaleByPowerOfTen(Math.abs(dec.scale())); + String digits = dec.unscaledValue().toString(); + + // finally write out the pieces (sign, scale, digits) + buffer.write((byte) ( sign + 1), invert); + buffer.write((byte) ((factor >> 24) ^ 0x80), invert); + buffer.write((byte) ( factor >> 16), invert); + buffer.write((byte) ( factor >> 8), invert); + buffer.write((byte) factor, invert); + serializeBytes(buffer, digits.getBytes(decimalCharSet), + digits.length(), sign == -1 ? !invert : invert); + return; + } + default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); diff --git serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java new file mode 100644 index 0000000..1d0a31b --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.ByteStream.Output; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +public class BigDecimalWritable implements WritableComparable { + + static final private Log LOG = LogFactory.getLog(BigDecimalWritable.class); + + private byte[] internalStorage = new byte[0]; + private int scale; + + private final VInt vInt = new VInt(); // reusable integer + + public BigDecimalWritable() { + } + + public BigDecimalWritable(byte[] bytes, int scale) { + set(bytes, scale); + } + + public BigDecimalWritable(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public BigDecimalWritable(BigDecimal value) { + set(value); + } + + public void set(BigDecimal value) { + value = value.stripTrailingZeros(); + set(value.unscaledValue().toByteArray(), value.scale()); + } + + public void set(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public void set(byte[] bytes, int scale) { + this.internalStorage = bytes; + this.scale = scale; + } + + public void setFromBytes(byte[] bytes, int offset, int length) { + LazyBinaryUtils.readVInt(bytes, offset, vInt); + scale = vInt.value; + offset += vInt.length; + LazyBinaryUtils.readVInt(bytes, offset, vInt); + offset += vInt.length; + if (internalStorage.length != vInt.value) { + internalStorage = new byte[vInt.value]; + } + System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); + } + + public BigDecimal getBigDecimal() { + return new BigDecimal(new BigInteger(internalStorage), scale); + } + + @Override + public void readFields(DataInput in) throws IOException { + scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + if (internalStorage.length != byteArrayLen) { + internalStorage = new byte[byteArrayLen]; + } + in.readFully(internalStorage); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, scale); + WritableUtils.writeVInt(out, internalStorage.length); + out.write(internalStorage); + } + + @Override + public int compareTo(BigDecimalWritable that) { + return getBigDecimal().compareTo(that.getBigDecimal()); + } + + public void writeToByteStream(Output byteStream) { + LazyBinaryUtils.writeVInt(byteStream, scale); + LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); + byteStream.write(internalStorage, 0, internalStorage.length); + } + + @Override + public String toString() { + return getBigDecimal().toString(); + } + + @Override + public boolean equals(Object other) { + if (other == null || !(other instanceof BigDecimalWritable)) { + return false; + } + BigDecimalWritable bdw = (BigDecimalWritable) other; + + // 'equals' and 'compareTo' are not compatible with BigDecimals. We want + // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is + // the same as 3.140). 'Equals' returns true iff equal and the same scale + // is set in the decimals (e.g.: 3.14 is not the same as 3.140) + return getBigDecimal().compareTo(bdw.getBigDecimal()) == 0; + } + + @Override + public int hashCode() { + return getBigDecimal().hashCode(); + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java new file mode 100644 index 0000000..4f4f1f3 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java @@ -0,0 +1,54 @@ +package org.apache.hadoop.hive.serde2.lazy; + +import java.math.BigDecimal; +import java.nio.charset.CharacterCodingException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBigDecimalObjectInspector; +import org.apache.hadoop.io.Text; + +public class LazyBigDecimal extends LazyPrimitive { + static final private Log LOG = LogFactory.getLog(LazyBigDecimal.class); + + public LazyBigDecimal(LazyBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + public LazyBigDecimal(LazyBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + /** + * Initilizes LazyBigDecimal object by interpreting the input bytes + * as a numeric string + * + * @param bytes + * @param start + * @param length + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; + try { + byteData = Text.decode(bytes.getData(), start, length); + data.set(new BigDecimal(byteData)); + isNull = false; + } catch (NumberFormatException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null. Given data is :" + + byteData, e); + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null.", e); + } + } + + @Override + public BigDecimalWritable getWritableObject() { + return data; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 20758a7..2c6251f 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -26,8 +26,9 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector; @@ -112,6 +113,8 @@ return new LazyTimestamp((LazyTimestampObjectInspector) oi); case BINARY: return new LazyBinary((LazyBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBigDecimal((LazyBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyObject for " + p); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index c96f5b4..b93709e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -21,6 +21,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -235,6 +237,12 @@ public static void writePrimitiveUTF8(OutputStream out, Object o, ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break; } + case DECIMAL: { + BigDecimal bd = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + ByteBuffer b = Text.encode(bd.toString()); + out.write(b.array(), 0, b.limit()); + break; + } default: { throw new RuntimeException("Hive internal error."); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java new file mode 100644 index 0000000..15eb0f6 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.LazyBigDecimal; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public class LazyBigDecimalObjectInspector + extends AbstractPrimitiveLazyObjectInspector + implements BigDecimalObjectInspector { + + protected LazyBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new LazyBigDecimal((LazyBigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((LazyBigDecimal) o).getWritableObject().getBigDecimal(); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index 57d2fad..1e0ad00 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -57,6 +57,8 @@ new LazyTimestampObjectInspector(); public static final LazyBinaryObjectInspector LAZY_BINARY_OBJECT_INSPECTOR = new LazyBinaryObjectInspector(); + public static final LazyBigDecimalObjectInspector LAZY_BIG_DECIMAL_OBJECT_INSPECTOR = + new LazyBigDecimalObjectInspector(); static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>(); @@ -101,6 +103,8 @@ public static LazyStringObjectInspector getLazyStringObjectInspector( return LAZY_VOID_OBJECT_INSPECTOR; case TIMESTAMP: return LAZY_TIMESTAMP_OBJECT_INSPECTOR; + case DECIMAL: + return LAZY_BIG_DECIMAL_OBJECT_INSPECTOR; default: throw new RuntimeException("Internal error: Cannot find ObjectInspector " + " for " + primitiveCategory); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java new file mode 100644 index 0000000..5d8a6a1 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazybinary; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; + +public class LazyBinaryBigDecimal extends + LazyBinaryPrimitive { + + LazyBinaryBigDecimal(WritableBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + LazyBinaryBigDecimal(LazyBinaryBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + data.setFromBytes(bytes.getData(), start, length); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java index 86f098f..3111cbc 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector; @@ -75,6 +76,8 @@ return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi); case BINARY: return new LazyBinaryBinary((WritableBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBinaryBigDecimal((WritableBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyBinaryObject for " + p); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index c640d6a..ec9c88a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -382,6 +384,14 @@ public static boolean serialize(Output byteStream, Object obj, t.writeToByteStream(byteStream); return warnedOnceNullMapKey; } + + case DECIMAL: { + BigDecimalObjectInspector bdoi = (BigDecimalObjectInspector) poi; + BigDecimalWritable t = bdoi.getPrimitiveWritableObject(obj); + t.writeToByteStream(byteStream); + return warnedOnceNullMapKey; + } + default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index e024cb9..cac20ff 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -203,6 +203,14 @@ public static void checkObjectByteInfo(ObjectInspector objectInspector, recordInfo.elementSize += (byte) WritableUtils.decodeVIntSize(bytes[offset+4]); } break; + case DECIMAL: + // using vint instead of 4 bytes + LazyBinaryUtils.readVInt(bytes, offset, vInt); + recordInfo.elementOffset = 0; + recordInfo.elementSize = vInt.length; + LazyBinaryUtils.readVInt(bytes, offset + vInt.length, vInt); + recordInfo.elementSize += vInt.length + vInt.value; + break; default: { throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 5392b0e..6070afb 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; @@ -118,6 +119,10 @@ public static Converter getConverter(ObjectInspector inputOI, return new PrimitiveObjectInspectorConverter.BinaryConverter( (PrimitiveObjectInspector)inputOI, (SettableBinaryObjectInspector)outputOI); + case DECIMAL: + return new PrimitiveObjectInspectorConverter.BigDecimalConverter( + (PrimitiveObjectInspector) inputOI, + (SettableBigDecimalObjectInspector) outputOI); default: throw new RuntimeException("Hive internal error: conversion of " diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index 975255f..67132fc 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -30,10 +30,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -490,6 +492,9 @@ public static int hashCode(Object o, ObjectInspector objIns) { TimestampWritable t = ((TimestampObjectInspector) poi) .getPrimitiveWritableObject(o); return t.hashCode(); + case DECIMAL: + return ((BigDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); + default: { throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory()); @@ -674,6 +679,13 @@ public static int compare(Object o1, ObjectInspector oi1, Object o2, .getPrimitiveWritableObject(o2); return t1.compareTo(t2); } + case DECIMAL: { + BigDecimalWritable t1 = ((BigDecimalObjectInspector) poi1) + .getPrimitiveWritableObject(o1); + BigDecimalWritable t2 = ((BigDecimalObjectInspector) poi2) + .getPrimitiveWritableObject(o2); + return t1.compareTo(t2); + } default: { throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory()); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java index 970e884..aace3bb 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -27,7 +27,7 @@ * The primitive types supported by Hive. */ public static enum PrimitiveCategory { - VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, UNKNOWN + VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, UNKNOWN }; /** diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java new file mode 100644 index 0000000..44db243 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +/** + * A DecimalObjectInspector inspects an Object representing a BigDecimal. + */ +public interface BigDecimalObjectInspector extends PrimitiveObjectInspector { + + BigDecimalWritable getPrimitiveWritableObject(Object o); + + BigDecimal getPrimitiveJavaObject(Object o); +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java new file mode 100644 index 0000000..382d7e8 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class JavaBigDecimalObjectInspector + extends AbstractPrimitiveJavaObjectInspector + implements SettableBigDecimalObjectInspector { + + protected JavaBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : (BigDecimal) o; + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object set(Object o, BigDecimal t) { + return t; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + return t == null ? null : t.getBigDecimal(); + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object create(BigDecimal t) { + return t == null ? null : new BigDecimal(t.unscaledValue(), t.scale()); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index f0b16fa..1c4c8bd 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; +import java.math.BigDecimal; import java.sql.Timestamp; import org.apache.hadoop.hive.serde2.ByteStream; @@ -257,6 +258,30 @@ public Object convert(Object input) { } } + public static class BigDecimalConverter implements Converter { + + PrimitiveObjectInspector inputOI; + SettableBigDecimalObjectInspector outputOI; + Object r; + + public BigDecimalConverter(PrimitiveObjectInspector inputOI, + SettableBigDecimalObjectInspector outputOI) { + this.inputOI = inputOI; + this.outputOI = outputOI; + this.r = outputOI.create(BigDecimal.ZERO); + } + + @Override + public Object convert(Object input) { + if (input == null) { + return null; + } + return outputOI.set(r, PrimitiveObjectInspectorUtils.getBigDecimal(input, + inputOI)); + } + + } + public static class BinaryConverter implements Converter{ PrimitiveObjectInspector inputOI; @@ -333,11 +358,11 @@ public Text convert(Object input) { t.set(String.valueOf(((DoubleObjectInspector) inputOI).get(input))); return t; case STRING: - if (inputOI.preferWritable()) { - t.set(((StringObjectInspector) inputOI).getPrimitiveWritableObject(input)); - } else { - t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input)); - } + if (inputOI.preferWritable()) { + t.set(((StringObjectInspector) inputOI).getPrimitiveWritableObject(input)); + } else { + t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input)); + } return t; case TIMESTAMP: t.set(((TimestampObjectInspector) inputOI) @@ -346,6 +371,9 @@ public Text convert(Object input) { case BINARY: t.set(((BinaryObjectInspector) inputOI).getPrimitiveWritableObject(input).getBytes()); return t; + case DECIMAL: + t.set(((BigDecimalObjectInspector) inputOI).getPrimitiveWritableObject(input).toString()); + return t; default: throw new RuntimeException("Hive 2 Internal error: type = " + inputOI.getTypeName()); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 7f61344..a39934c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -20,6 +20,7 @@ import java.util.HashMap; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -69,6 +70,8 @@ new JavaTimestampObjectInspector(); public static final JavaBinaryObjectInspector javaByteArrayObjectInspector = new JavaBinaryObjectInspector(); + public static final JavaBigDecimalObjectInspector javaBigDecimalObjectInspector = + new JavaBigDecimalObjectInspector(); public static final WritableBooleanObjectInspector writableBooleanObjectInspector = new WritableBooleanObjectInspector(); @@ -92,6 +95,8 @@ new WritableTimestampObjectInspector(); public static final WritableBinaryObjectInspector writableBinaryObjectInspector = new WritableBinaryObjectInspector(); + public static final WritableBigDecimalObjectInspector writableBigDecimalObjectInspector = + new WritableBigDecimalObjectInspector(); private static HashMap cachedPrimitiveWritableInspectorCache = new HashMap(); @@ -118,6 +123,8 @@ writableTimestampObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BINARY, writableBinaryObjectInspector); + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DECIMAL, + writableBigDecimalObjectInspector); } private static HashMap cachedPrimitiveJavaInspectorCache = @@ -145,6 +152,8 @@ javaTimestampObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BINARY, javaByteArrayObjectInspector); + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DECIMAL, + javaBigDecimalObjectInspector); } /** @@ -191,6 +200,8 @@ public static ConstantObjectInspector getPrimitiveWritableConstantObjectInspecto return new WritableConstantStringObjectInspector((Text)value); case TIMESTAMP: return new WritableConstantTimestampObjectInspector((TimestampWritable)value); + case DECIMAL: + return new WritableConstantBigDecimalObjectInspector((BigDecimalWritable)value); case BINARY: return new WritableConstantBinaryObjectInspector((BytesWritable)value); case VOID: diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 84c642b..1860424 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -21,11 +21,13 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.math.BigDecimal; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -58,7 +60,7 @@ /** * TypeEntry stores information about a Hive Primitive TypeInfo. */ - public static class PrimitiveTypeEntry implements Writable{ + public static class PrimitiveTypeEntry implements Writable { /** * The category of the PrimitiveType. @@ -175,9 +177,13 @@ static void registerType(PrimitiveTypeEntry t) { public static final PrimitiveTypeEntry shortTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.SHORT, serdeConstants.SMALLINT_TYPE_NAME, Short.TYPE, Short.class, ShortWritable.class); + public static final PrimitiveTypeEntry timestampTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, TimestampWritable.class); + public static final PrimitiveTypeEntry decimalTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.DECIMAL, serdeConstants.DECIMAL_TYPE_NAME, null, + BigDecimal.class, BigDecimalWritable.class); // The following is a complex type for special handling public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( @@ -195,6 +201,7 @@ static void registerType(PrimitiveTypeEntry t) { registerType(byteTypeEntry); registerType(shortTypeEntry); registerType(timestampTypeEntry); + registerType(decimalTypeEntry); registerType(unknownTypeEntry); } @@ -358,10 +365,14 @@ public static boolean comparePrimitiveObjects(Object o1, return ((TimestampObjectInspector) oi1).getPrimitiveWritableObject(o1) .equals(((TimestampObjectInspector) oi2).getPrimitiveWritableObject(o2)); } - case BINARY:{ + case BINARY: { return ((BinaryObjectInspector) oi1).getPrimitiveWritableObject(o1). equals(((BinaryObjectInspector) oi2).getPrimitiveWritableObject(o2)); } + case DECIMAL: { + return ((BigDecimalObjectInspector) oi1).getPrimitiveJavaObject(o1) + .compareTo(((BigDecimalObjectInspector) oi2).getPrimitiveJavaObject(o2)) == 0; + } default: return false; } @@ -391,6 +402,8 @@ public static double convertPrimitiveToDouble(Object o, PrimitiveObjectInspector case TIMESTAMP: return ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); + case DECIMAL: + return ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o).doubleValue(); default: throw new NumberFormatException(); } @@ -465,6 +478,10 @@ public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) { result = (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds() != 0); break; + case DECIMAL: + result = BigDecimal.ZERO.compareTo( + ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)) != 0; + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -545,6 +562,10 @@ public static int getInt(Object o, PrimitiveObjectInspector oi) { result = (int) (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds()); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).intValue(); + break; default: { throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -599,6 +620,10 @@ public static long getLong(Object o, PrimitiveObjectInspector oi) { result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).longValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -646,6 +671,10 @@ public static double getDouble(Object o, PrimitiveObjectInspector oi) { case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getDouble(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).doubleValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -706,6 +735,10 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).toString(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -713,13 +746,13 @@ public static String getString(Object o, PrimitiveObjectInspector oi) { return result; } - public static BytesWritable getBinary(Object o, PrimitiveObjectInspector oi){ + public static BytesWritable getBinary(Object o, PrimitiveObjectInspector oi) { - if(null == o){ + if (null == o) { return null; } - switch (oi.getPrimitiveCategory()){ + switch (oi.getPrimitiveCategory()) { case VOID: return null; @@ -739,6 +772,58 @@ public static BytesWritable getBinary(Object o, PrimitiveObjectInspector oi){ } } + public static BigDecimal getBigDecimal(Object o, PrimitiveObjectInspector oi) { + if (o == null) { + return null; + } + + BigDecimal result = null; + switch (oi.getPrimitiveCategory()) { + case VOID: + result = null; + break; + case BOOLEAN: + result = ((BooleanObjectInspector) oi).get(o) ? + BigDecimal.ONE : BigDecimal.ZERO; + break; + case BYTE: + result = new BigDecimal(((ByteObjectInspector) oi).get(o)); + break; + case SHORT: + result = new BigDecimal(((ShortObjectInspector) oi).get(o)); + break; + case INT: + result = new BigDecimal(((IntObjectInspector) oi).get(o)); + break; + case LONG: + result = new BigDecimal(((LongObjectInspector) oi).get(o)); + break; + case FLOAT: + Float f = ((FloatObjectInspector) oi).get(o); + result = new BigDecimal(f.toString()); + break; + case DOUBLE: + Double d = ((DoubleObjectInspector) oi).get(o); + result = new BigDecimal(d.toString()); + break; + case STRING: + result = new BigDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + case TIMESTAMP: + Double ts = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) + .getDouble(); + result = new BigDecimal(ts.toString()); + break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + break; + default: + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + return result; + } + public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { if (o == null) { return null; @@ -801,7 +886,7 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { if (oi.getCategory() != Category.PRIMITIVE) { return null; } - PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi; + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; PrimitiveTypeEntry t = getTypeEntryFromPrimitiveCategory(poi.getPrimitiveCategory()); return t == null ? null : t.primitiveJavaClass; diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java new file mode 100644 index 0000000..ff262b2 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +/** + * A SettableDecimalObjectInspector can set a BigDecimal value to an object. + */ +public interface SettableBigDecimalObjectInspector extends BigDecimalObjectInspector { + + Object set(Object o, byte[] bytes, int scale); + + Object set(Object o, BigDecimal t); + + Object set(Object o, BigDecimalWritable t); + + Object create(byte[] bytes, int scale); + + Object create (BigDecimal t); + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java new file mode 100644 index 0000000..88184cf --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class WritableBigDecimalObjectInspector + extends AbstractPrimitiveWritableObjectInspector + implements SettableBigDecimalObjectInspector { + + protected WritableBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : (BigDecimalWritable) o; + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((BigDecimalWritable) o).getBigDecimal(); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimalWritable) o); + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + ((BigDecimalWritable) o).set(bytes, scale); + return o; + } + + @Override + public Object set(Object o, BigDecimal t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimalWritable(bytes, scale); + } + + @Override + public Object create(BigDecimal t) { + return new BigDecimalWritable(t); + } + +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java new file mode 100644 index 0000000..672b106 --- /dev/null +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +/** + * A WritableConstantBigDecimalObjectInspector is a WritableBigDecimalObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantBigDecimalObjectInspector extends WritableBigDecimalObjectInspector + implements ConstantObjectInspector { + + private final BigDecimalWritable value; + + WritableConstantBigDecimalObjectInspector(BigDecimalWritable value) { + this.value = value; + } + + @Override + public BigDecimalWritable getWritableConstantValue() { + return value; + } +} diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 1f0a578..c766c9e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -64,6 +64,7 @@ public static TypeInfo getPrimitiveTypeInfo(String typeName) { public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); + public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(serdeConstants.DECIMAL_TYPE_NAME); public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); diff --git serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java index 485919f..095387f 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java +++ serde/src/test/org/apache/hadoop/hive/serde2/TestStatsSerde.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.serde2; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.List; import java.util.Properties; import java.util.Random; @@ -103,7 +105,7 @@ public void testLazyBinarySerDe() throws Throwable { Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; for (int i = 0; i < num; i++) { - int randField = r.nextInt(10); + int randField = r.nextInt(12); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -112,12 +114,13 @@ public void testLazyBinarySerDe() throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + BigDecimal bd = randField > 8 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); + MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 8 ? null : TestBinarySortableSerDe + List li = randField > 10 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, i); - MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, is, li,ba); + MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, is, li,ba); rows[i] = t; } diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java index b6f99d3..bf7b265 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java @@ -17,34 +17,37 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; +import java.math.BigDecimal; import java.util.List; public class MyTestClass { - Byte myByte; - Short myShort; - Integer myInt; - Long myLong; - Float myFloat; - Double myDouble; - String myString; - MyTestInnerStruct myStruct; - List myList; - byte[] myBA; + Byte myByte; + Short myShort; + Integer myInt; + Long myLong; + Float myFloat; + Double myDouble; + String myString; + BigDecimal myDecimal; + MyTestInnerStruct myStruct; + List myList; + byte[] myBA; - public MyTestClass() { - } + public MyTestClass() { + } - public MyTestClass(Byte b, Short s, Integer i, Long l, Float f, Double d, - String st, MyTestInnerStruct is, List li, byte[] ba) { - myByte = b; - myShort = s; - myInt = i; - myLong = l; - myFloat = f; - myDouble = d; - myString = st; - myStruct = is; - myList = li; - myBA = ba; - } + public MyTestClass(Byte b, Short s, Integer i, Long l, Float f, Double d, + String st, BigDecimal bd, MyTestInnerStruct is, List li, byte[] ba) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myDecimal = bd; + myStruct = is; + myList = li; + myBA = ba; + } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java index 0889ef5..a6af537 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -42,6 +43,8 @@ */ public class TestBinarySortableSerDe extends TestCase { + private static final String DECIMAL_CHARS = "0123456789"; + public static HashMap makeHashMap(String... params) { HashMap r = new HashMap(); for (int i = 0; i < params.length; i += 2) { @@ -132,11 +135,36 @@ private void sort(Object[] structs, ObjectInspector oi) { } } + public static BigDecimal getRandBigDecimal(Random r) { + StringBuilder sb = new StringBuilder(); + int l1 = 1+r.nextInt(500), l2 = r.nextInt(500); + + if (r.nextBoolean()) { + sb.append("-"); + } + + sb.append(getRandString(r, DECIMAL_CHARS, l1)); + if (l2 != 0) { + sb.append("."); + sb.append(getRandString(r, DECIMAL_CHARS, l2)); + } + + BigDecimal bd = new BigDecimal(sb.toString()); + return bd; + } + public static String getRandString(Random r) { - int length = r.nextInt(10); + return getRandString(r, null, r.nextInt(10)); + } + + public static String getRandString(Random r, String characters, int length) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < length; i++) { - sb.append((char) (r.nextInt(128))); + if (characters == null) { + sb.append((char) (r.nextInt(128))); + } else { + sb.append(characters.charAt(r.nextInt(characters.length()))); + } } return sb.toString(); } @@ -179,9 +207,10 @@ public void testBinarySortableSerDe() throws Throwable { t.myDouble = randField > 5 ? null : Double .valueOf(r.nextDouble() * 10 - 5); t.myString = randField > 6 ? null : getRandString(r); - t.myStruct = randField > 7 ? null : new MyTestInnerStruct( + t.myDecimal = randField > 7 ? null : getRandBigDecimal(r); + t.myStruct = randField > 8 ? null : new MyTestInnerStruct( r.nextInt(5) - 2, r.nextInt(5) - 2); - t.myList = randField > 8 ? null : getRandIntegerArray(r); + t.myList = randField > 9 ? null : getRandIntegerArray(r); t.myBA = getRandBA(r, i); rows[i] = t; } @@ -195,9 +224,9 @@ public void testBinarySortableSerDe() throws Throwable { String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI); testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, - "++++++++++"), true); + "+++++++++++"), true); testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, - "----------"), false); + "-----------"), false); System.out.println("Test testTBinarySortableProtocol passed!"); } catch (Throwable e) { diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java index 7050def..75a2b57 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -27,34 +28,36 @@ * */ public class MyTestClassBigger { - Byte myByte; - Short myShort; - Integer myInt; - Long myLong; - Float myFloat; - Double myDouble; - String myString; - MyTestInnerStruct myStruct; - List myList; - byte[] myBA; - Map> myMap; + Byte myByte; + Short myShort; + Integer myInt; + Long myLong; + Float myFloat; + Double myDouble; + String myString; + BigDecimal myDecimal; + MyTestInnerStruct myStruct; + List myList; + byte[] myBA; + Map> myMap; - public MyTestClassBigger() { - } + public MyTestClassBigger() { + } - public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f, - Double d, String st, MyTestInnerStruct is, List li, - byte[] ba, Map> mp) { - myByte = b; - myShort = s; - myInt = i; - myLong = l; - myFloat = f; - myDouble = d; - myString = st; - myStruct = is; - myList = li; - myBA = ba; - myMap = mp; - } + public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f, + Double d, String st, BigDecimal bd, MyTestInnerStruct is, List li, + byte[] ba, Map> mp) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myDecimal = bd; + myStruct = is; + myList = li; + myBA = ba; + myMap = mp; + } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java index 29f0601..f692d99 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java @@ -17,30 +17,33 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.math.BigDecimal; import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct; public class MyTestClassSmaller { - Byte myByte; - Short myShort; - Integer myInt; - Long myLong; - Float myFloat; - Double myDouble; - String myString; - MyTestInnerStruct myStruct; + Byte myByte; + Short myShort; + Integer myInt; + Long myLong; + Float myFloat; + Double myDouble; + String myString; + BigDecimal myDecimal; + MyTestInnerStruct myStruct; - public MyTestClassSmaller() { - } - - public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f, - Double d, String st, MyTestInnerStruct is) { - myByte = b; - myShort = s; - myInt = i; - myLong = l; - myFloat = f; - myDouble = d; - myString = st; - myStruct = is; - } + public MyTestClassSmaller() { + } + + public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f, + Double d, String st, BigDecimal bd, MyTestInnerStruct is) { + myByte = b; + myShort = s; + myInt = i; + myLong = l; + myFloat = f; + myDouble = d; + myString = st; + myDecimal = bd; + myStruct = is; + } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java index 9b38679..030bc09 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -191,7 +192,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(11); + int randField = r.nextInt(10); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -200,6 +201,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); + BigDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); List li = randField > 8 ? null : TestBinarySortableSerDe @@ -207,7 +209,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); Map> mp = new HashMap>(); String key = TestBinarySortableSerDe.getRandString(r); - List value = randField > 10 ? null + List value = randField > 9 ? null : getRandStructArray(r); mp.put(key, value); String key1 = TestBinarySortableSerDe.getRandString(r); @@ -216,7 +218,7 @@ private void testShorterSchemaDeserialization(Random r) throws Throwable { List value2 = getRandStructArray(r); mp.put(key2, value2); - MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, is, + MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, bd, is, li, ba, mp); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -258,7 +260,7 @@ private void testShorterSchemaDeserialization1(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(10); + int randField = r.nextInt(11); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -267,12 +269,13 @@ private void testShorterSchemaDeserialization1(Random r) throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + BigDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); + MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 8 ? null : TestBinarySortableSerDe + List li = randField > 9 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); - MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li, ba); + MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, is, li, ba); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -313,7 +316,7 @@ void testLongerSchemaDeserialization(Random r) throws Throwable { int num = 100; for (int itest = 0; itest < num; itest++) { - int randField = r.nextInt(10); + int randField = r.nextInt(11); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -322,12 +325,13 @@ void testLongerSchemaDeserialization(Random r) throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + BigDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); + MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 8 ? null : TestBinarySortableSerDe + List li = randField > 9 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest); - MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li,ba); + MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, is, li,ba); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -377,10 +381,11 @@ void testLongerSchemaDeserialization1(Random r) throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); + BigDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, + MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, is); BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1); Object output = serde2.deserialize(bw); @@ -410,13 +415,13 @@ void testLazyBinaryMap(Random r) throws Throwable { StructObjectInspector soi1 = (StructObjectInspector) serdeOI; List fields1 = soi1.getAllStructFieldRefs(); LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1 - .get(10).getFieldObjectInspector(); + .get(11).getFieldObjectInspector(); ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector(); ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector(); StructObjectInspector soi2 = rowOI; List fields2 = soi2.getAllStructFieldRefs(); - MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(10) + MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(11) .getFieldObjectInspector(); ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector(); ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector(); @@ -436,10 +441,10 @@ void testLazyBinaryMap(Random r) throws Throwable { } MyTestClassBigger input = new MyTestClassBigger(null, null, null, null, - null, null, null, null, null, null, mp); + null, null, null, null, null, null, null, mp); BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI); Object output = serde.deserialize(bw); - Object lazyobj = soi1.getStructFieldData(output, fields1.get(10)); + Object lazyobj = soi1.getStructFieldData(output, fields1.get(11)); Map outputmp = lazympoi.getMap(lazyobj); if (outputmp.size() != mp.size()) { @@ -486,7 +491,7 @@ public void testLazyBinarySerDe() throws Throwable { Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; for (int i = 0; i < num; i++) { - int randField = r.nextInt(10); + int randField = r.nextInt(11); Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt()); Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt()); Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt()); @@ -495,12 +500,13 @@ public void testLazyBinarySerDe() throws Throwable { Double d = randField > 5 ? null : Double.valueOf(r.nextDouble()); String st = randField > 6 ? null : TestBinarySortableSerDe .getRandString(r); - MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r + BigDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandBigDecimal(r); + MyTestInnerStruct is = randField > 8 ? null : new MyTestInnerStruct(r .nextInt(5) - 2, r.nextInt(5) - 2); - List li = randField > 8 ? null : TestBinarySortableSerDe + List li = randField > 9 ? null : TestBinarySortableSerDe .getRandIntegerArray(r); byte[] ba = TestBinarySortableSerDe.getRandBA(r, i); - MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, is, li, ba); + MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, is, li, ba); rows[i] = t; } diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java index e221712..6fda3c2 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java @@ -19,6 +19,8 @@ import junit.framework.TestCase; +import java.math.BigDecimal; + import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -126,6 +128,11 @@ public void testObjectInspectorConverters() throws Throwable { PrimitiveObjectInspectorFactory.writableStringObjectInspector); assertEquals("TextConverter", new Text("hive"), textConverter .convert(new String("hive"))); + textConverter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaBigDecimalObjectInspector, + PrimitiveObjectInspectorFactory.writableStringObjectInspector); + assertEquals("TextConverter", new Text("100.001"), textConverter + .convert(new BigDecimal("100.001"))); // Binary Converter baConverter = ObjectInspectorConverters.getConverter(