From 3edd8a69516d336cf5ea7bec9cfbb15bf067c6b8 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 25 Oct 2012 19:49:37 -0700 Subject: [PATCH 1/4] HIVE-2693: Add BigDecimalWritable class --- .../hadoop/hive/serde2/io/BigDecimalWritable.java | 119 ++++++++++++++++++++ 1 files changed, 119 insertions(+), 0 deletions(-) create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java new file mode 100644 index 0000000..df8166b --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java @@ -0,0 +1,119 @@ ++/** ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * "License"); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.hadoop.hive.serde2.io; ++ ++import java.io.DataInput; ++import java.io.DataOutput; ++import java.io.IOException; ++import java.math.BigDecimal; ++import java.math.BigInteger; ++ ++import org.apache.commons.logging.Log; ++import org.apache.commons.logging.LogFactory; ++import org.apache.hadoop.hive.serde2.ByteStream.Output; ++import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; ++import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; ++import org.apache.hadoop.io.WritableComparable; ++import org.apache.hadoop.io.WritableUtils; ++ ++public class BigDecimalWritable implements WritableComparable { ++ ++ static final private Log LOG = LogFactory.getLog(BigDecimalWritable.class); ++ ++ private byte[] internalStorage = new byte[0]; ++ private int scale; ++ ++ public BigDecimalWritable() { ++ } ++ ++ public BigDecimalWritable(byte[] bytes, int scale) { ++ set(bytes, scale); ++ } ++ ++ public BigDecimalWritable(BigDecimalWritable writable) { ++ set(writable.getBigDecimal()); ++ } ++ ++ public BigDecimalWritable(BigDecimal value) { ++ set(value); ++ } ++ ++ public void set(BigDecimal value) { ++ set(value.unscaledValue().toByteArray(), value.scale()); ++ } ++ ++ public void set(BigDecimalWritable writable) { ++ set(writable.getBigDecimal()); ++ } ++ ++ public void set(byte[] bytes, int scale) { ++ this.internalStorage = bytes; ++ this.scale = scale; ++ } ++ ++ private final VInt vInt = new VInt(); ++ ++ public void setFromBytes(byte[] bytes, int offset, int length) { ++ LazyBinaryUtils.readVInt(bytes, offset, vInt); ++ scale = vInt.value; ++ offset += vInt.length; ++ LazyBinaryUtils.readVInt(bytes, offset, vInt); ++ offset += vInt.length; ++ if (internalStorage.length != vInt.value) { ++ internalStorage = new byte[vInt.value]; ++ } ++ System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); ++ } ++ ++ public BigDecimal getBigDecimal() { ++ return new BigDecimal(new BigInteger(internalStorage), scale); ++ } ++ ++ @Override ++ public void readFields(DataInput in) throws IOException { ++ scale = WritableUtils.readVInt(in); ++ int byteArrayLen = WritableUtils.readVInt(in); ++ if (internalStorage.length != byteArrayLen) { ++ internalStorage = new byte[byteArrayLen]; ++ } ++ in.readFully(internalStorage); ++ } ++ ++ @Override ++ public void write(DataOutput out) throws IOException { ++ WritableUtils.writeVInt(out, scale); ++ WritableUtils.writeVInt(out, internalStorage.length); ++ out.write(internalStorage); ++ } ++ ++ @Override ++ public int compareTo(BigDecimalWritable that) { ++ return getBigDecimal().compareTo(that.getBigDecimal()); ++ } ++ ++ public void writeToByteStream(Output byteStream) { ++ LazyBinaryUtils.writeVInt(byteStream, scale); ++ LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); ++ byteStream.write(internalStorage, 0, internalStorage.length); ++ } ++ ++ @Override ++ public String toString() { ++ return getBigDecimal().toString(); ++ } ++} -- 1.7.3.4 From 741e49f43c5ee53bf98771983b7d28ab20170cc1 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 25 Oct 2012 20:01:11 -0700 Subject: [PATCH 2/4] HIVE-2693: Omnibus patch to add a DECIMAL type to Hive --- .../hadoop/hive/metastore/MetaStoreUtils.java | 4 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 10 +- .../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 1 + ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g | 3 + .../hadoop/hive/ql/parse/TypeCheckProcFactory.java | 2 + .../java/org/apache/hadoop/hive/ql/udf/UDFAbs.java | 11 + .../hadoop/hive/ql/udf/UDFBaseNumericOp.java | 3 + .../hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java | 3 + .../org/apache/hadoop/hive/ql/udf/UDFCeil.java | 19 ++- .../java/org/apache/hadoop/hive/ql/udf/UDFExp.java | 11 +- .../org/apache/hadoop/hive/ql/udf/UDFFloor.java | 19 ++- .../java/org/apache/hadoop/hive/ql/udf/UDFLog.java | 31 +++- .../org/apache/hadoop/hive/ql/udf/UDFLog10.java | 19 ++- .../org/apache/hadoop/hive/ql/udf/UDFLog2.java | 16 ++- .../org/apache/hadoop/hive/ql/udf/UDFOPDivide.java | 15 +- .../org/apache/hadoop/hive/ql/udf/UDFOPMinus.java | 12 + .../org/apache/hadoop/hive/ql/udf/UDFOPMod.java | 11 + .../apache/hadoop/hive/ql/udf/UDFOPMultiply.java | 11 + .../apache/hadoop/hive/ql/udf/UDFOPNegative.java | 10 + .../org/apache/hadoop/hive/ql/udf/UDFOPPlus.java | 11 + .../apache/hadoop/hive/ql/udf/UDFOPPositive.java | 6 + .../org/apache/hadoop/hive/ql/udf/UDFPosMod.java | 15 ++ .../org/apache/hadoop/hive/ql/udf/UDFPower.java | 9 +- .../org/apache/hadoop/hive/ql/udf/UDFRound.java | 17 ++- .../org/apache/hadoop/hive/ql/udf/UDFSqrt.java | 16 ++- .../apache/hadoop/hive/ql/udf/UDFToBoolean.java | 14 +- .../org/apache/hadoop/hive/ql/udf/UDFToByte.java | 11 +- .../org/apache/hadoop/hive/ql/udf/UDFToDouble.java | 11 +- .../org/apache/hadoop/hive/ql/udf/UDFToFloat.java | 12 +- .../apache/hadoop/hive/ql/udf/UDFToInteger.java | 12 +- .../org/apache/hadoop/hive/ql/udf/UDFToLong.java | 12 +- .../org/apache/hadoop/hive/ql/udf/UDFToShort.java | 10 + .../org/apache/hadoop/hive/ql/udf/UDFToString.java | 12 +- .../ql/udf/generic/GenericUDAFCorrelation.java | 2 + .../hive/ql/udf/generic/GenericUDAFCovariance.java | 2 + .../udf/generic/GenericUDAFCovarianceSample.java | 2 + .../udf/generic/GenericUDAFHistogramNumeric.java | 9 +- .../udf/generic/GenericUDAFPercentileApprox.java | 21 +- .../hadoop/hive/ql/udf/generic/GenericUDAFStd.java | 5 +- .../hive/ql/udf/generic/GenericUDAFStdSample.java | 3 +- .../hadoop/hive/ql/udf/generic/GenericUDAFSum.java | 88 +++++++ .../hive/ql/udf/generic/GenericUDAFVariance.java | 11 +- .../ql/udf/generic/GenericUDAFVarianceSample.java | 3 +- .../hive/ql/udf/generic/GenericUDFToDecimal.java | 74 ++++++ ql/src/test/queries/clientpositive/decimal_1.q | 17 ++ ql/src/test/queries/clientpositive/decimal_2.q | 17 ++ ql/src/test/results/clientpositive/decimal_1.q.out | 117 ++++++++++ ql/src/test/results/clientpositive/decimal_2.q.out | 117 ++++++++++ .../org/apache/hadoop/hive/serde2/SerDeUtils.java | 5 + .../hadoop/hive/serde2/io/BigDecimalWritable.java | 238 ++++++++++---------- .../serde2/lazybinary/LazyBinaryBigDecimal.java | 42 ++++ .../hive/serde2/lazybinary/LazyBinaryFactory.java | 3 + .../hive/serde2/lazybinary/LazyBinarySerDe.java | 10 + .../hive/serde2/lazybinary/LazyBinaryUtils.java | 8 + .../primitive/PrimitiveObjectInspectorFactory.java | 11 + .../WritableConstantBigDecimalObjectInspector.java | 40 ++++ 56 files changed, 1055 insertions(+), 169 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java create mode 100644 ql/src/test/queries/clientpositive/decimal_1.q create mode 100644 ql/src/test/queries/clientpositive/decimal_2.q create mode 100644 ql/src/test/results/clientpositive/decimal_1.q.out create mode 100644 ql/src/test/results/clientpositive/decimal_2.q.out create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 1a15500..a83e2e0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -391,7 +391,7 @@ public class MetaStoreUtils { org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME, "string"); typeToThriftTypeMap.put( org.apache.hadoop.hive.serde.Constants.BINARY_TYPE_NAME, "binary"); - // These 3 types are not supported yet. + // These 4 types are not supported yet. // We should define a complex type date in thrift that contains a single int // member, and DynamicSerDe // should convert it to date type at runtime. @@ -402,6 +402,8 @@ public class MetaStoreUtils { typeToThriftTypeMap .put(org.apache.hadoop.hive.serde.Constants.TIMESTAMP_TYPE_NAME, "timestamp"); + typeToThriftTypeMap.put( + org.apache.hadoop.hive.serde.Constants.DECIMAL_TYPE_NAME, "decimal"); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 425900d..130d5b0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -169,8 +169,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInFile; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLocate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMap; @@ -198,6 +198,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStringToMap; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTranslate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion; @@ -227,10 +228,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.ReflectionUtils; - import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; /** @@ -402,6 +401,8 @@ public final class FunctionRegistry { GenericUDFTimestamp.class); registerGenericUDF(Constants.BINARY_TYPE_NAME, GenericUDFToBinary.class); + registerGenericUDF(Constants.DECIMAL_TYPE_NAME, + GenericUDFToDecimal.class); // Aggregate functions registerGenericUDAF("max", new GenericUDAFMax()); @@ -634,7 +635,8 @@ public final class FunctionRegistry { registerNumericType(Constants.BIGINT_TYPE_NAME, 4); registerNumericType(Constants.FLOAT_TYPE_NAME, 5); registerNumericType(Constants.DOUBLE_TYPE_NAME, 6); - registerNumericType(Constants.STRING_TYPE_NAME, 7); + registerNumericType(Constants.DECIMAL_TYPE_NAME, 7); + registerNumericType(Constants.STRING_TYPE_NAME, 8); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 5fc6a4f..466f7e9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -144,6 +144,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { TokenToTypeName.put(HiveParser.TOK_DATE, Constants.DATE_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DATETIME, Constants.DATETIME_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_TIMESTAMP, Constants.TIMESTAMP_TYPE_NAME); + TokenToTypeName.put(HiveParser.TOK_DECIMAL, Constants.DECIMAL_TYPE_NAME); } public static String getTypeName(int token) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g index 5884328..94e5d55 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g @@ -104,6 +104,7 @@ TOK_DATETIME; TOK_TIMESTAMP; TOK_STRING; TOK_BINARY; +TOK_DECIMAL; TOK_LIST; TOK_STRUCT; TOK_MAP; @@ -1371,6 +1372,7 @@ primitiveType | KW_TIMESTAMP -> TOK_TIMESTAMP | KW_STRING -> TOK_STRING | KW_BINARY -> TOK_BINARY + | KW_DECIMAL -> TOK_DECIMAL ; listType @@ -2304,6 +2306,7 @@ KW_DOUBLE: 'DOUBLE'; KW_DATE: 'DATE'; KW_DATETIME: 'DATETIME'; KW_TIMESTAMP: 'TIMESTAMP'; +KW_DECIMAL: 'DECIMAL'; KW_STRING: 'STRING'; KW_ARRAY: 'ARRAY'; KW_STRUCT: 'STRUCT'; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index d4f6a9f..ea22154 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -499,6 +499,8 @@ public final class TypeCheckProcFactory { Constants.BINARY_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMP, Constants.TIMESTAMP_TYPE_NAME); + conversionFunctionTextHashMap.put(HiveParser.TOK_DECIMAL, + Constants.DECIMAL_TYPE_NAME); } public static boolean isRedundantConversionFunction(ASTNode expr, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java index 41043bc..bfce482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -38,6 +39,7 @@ public class UDFAbs extends UDF { private final DoubleWritable resultDouble = new DoubleWritable(); private final LongWritable resultLong = new LongWritable(); private final IntWritable resultInt = new IntWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public DoubleWritable evaluate(DoubleWritable n) { if (n == null) { @@ -68,4 +70,13 @@ public class UDFAbs extends UDF { return resultInt; } + + public BigDecimalWritable evaluate(BigDecimalWritable n) { + if (n == null) { + return null; + } + + resultBigDecimal.set(n.getBigDecimal().abs()); + return resultBigDecimal; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java index 63d0255..14c16ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.NumericOpMethodResolver; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -48,6 +49,7 @@ public abstract class UDFBaseNumericOp extends UDF { protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a, ByteWritable b); @@ -61,4 +63,5 @@ public abstract class UDFBaseNumericOp extends UDF { public abstract DoubleWritable evaluate(DoubleWritable a, DoubleWritable b); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java index b220805..cb7dca4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -45,6 +46,7 @@ public abstract class UDFBaseNumericUnaryOp extends UDF { protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a); @@ -58,4 +60,5 @@ public abstract class UDFBaseNumericUnaryOp extends UDF { public abstract DoubleWritable evaluate(DoubleWritable a); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java index 01dd4d6..d40a1eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ import org.apache.hadoop.io.LongWritable; + " 0\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFCeil extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public UDFCeil() { } @@ -48,4 +54,15 @@ public class UDFCeil extends UDF { } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + MathContext mc = new MathContext( + bd.precision() - bd.scale(), RoundingMode.CEILING); + bigDecimalWritable.set(bd.round(mc)); + return bigDecimalWritable; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java index 35b0987..10ae445 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,7 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; extended = "Example:\n " + " > SELECT _FUNC_(0) FROM src LIMIT 1;\n" + " 1") public class UDFExp extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFExp() { } @@ -48,4 +49,12 @@ public class UDFExp extends UDF { } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + result.set(Math.exp(a.getBigDecimal().doubleValue())); + return result; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java index 510a161..397a86e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ import org.apache.hadoop.io.LongWritable; + " -1\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFFloor extends UDF { - private LongWritable result = new LongWritable(); + private final LongWritable result = new LongWritable(); + private final BigDecimalWritable bdResult = new BigDecimalWritable(); public UDFFloor() { } @@ -48,4 +54,15 @@ public class UDFFloor extends UDF { } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + MathContext mc = new MathContext( + bd.precision() - bd.scale(), RoundingMode.FLOOR); + bdResult.set(bd.round(mc)); + return bdResult; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java index 6aa7f6a..b153fbf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,7 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; extended = "Example:\n" + " > SELECT _FUNC_(13, 13) FROM src LIMIT 1;\n" + " 1") public class UDFLog extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog() { } @@ -48,6 +49,19 @@ public class UDFLog extends UDF { } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v)); + return result; + } + } + /** * Returns the logarithm of "a" with base "base". */ @@ -60,4 +74,19 @@ public class UDFLog extends UDF { } } + /** + * Returns the logarithm of "a" with base "base". + */ + public DoubleWritable evaluate(DoubleWritable base, BigDecimalWritable a) { + if (a == null || base == null || base.get() <= 1.0) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v <= 0.0) { + return null; + } + result.set(Math.log(v) / Math.log(base.get())); + return result; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java index a6e7c5e..d49384e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -33,7 +34,7 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; public class UDFLog10 extends UDF { private static double log10 = Math.log(10.0); - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog10() { } @@ -50,4 +51,20 @@ public class UDFLog10 extends UDF { } } + /** + * Returns the logarithm of "a" with base 10. + */ + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v) / log10); + return result; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java index 87c6d22..1d3af4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -33,7 +34,7 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; public class UDFLog2 extends UDF { private static double log2 = Math.log(2.0); - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog2() { } @@ -50,4 +51,17 @@ public class UDFLog2 extends UDF { } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v) / log2); + return result; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java index 0455aa9..a63ca55 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -29,12 +30,13 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; @Description(name = "/", value = "a _FUNC_ b - Divide a by b", extended = "Example:\n" + " > SELECT 3 _FUNC_ 2 FROM src LIMIT 1;\n" + " 1.5") /** - * Note that in SQL, the return type of divide is not necessarily the same + * Note that in SQL, the return type of divide is not necessarily the same * as the parameters. For example, 3 / 2 = 1.5, not 1. To follow SQL, we always * return a double for divide. */ public class UDFOPDivide extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":" @@ -46,4 +48,13 @@ public class UDFOPDivide extends UDF { doubleWritable.set(a.get() / b.get()); return doubleWritable; } + + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().divide(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java index 8ed1cc6..f884b9a 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,15 @@ public class UDFOPMinus extends UDFBaseNumericOp { doubleWritable.set(a.get() - b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().subtract(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java index 1935f03..95bbf45 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,14 @@ public class UDFOPMod extends UDFBaseNumericOp { doubleWritable.set(a.get() % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().remainder(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java index ce2c515..9058651 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,14 @@ public class UDFOPMultiply extends UDFBaseNumericOp { doubleWritable.set(a.get() * b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().multiply(b.getBigDecimal())); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java index 2378df2..3c14fef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -90,4 +91,13 @@ public class UDFOPNegative extends UDFBaseNumericUnaryOp { return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } + bigDecimalWritable.set(a.getBigDecimal().negate()); + return bigDecimalWritable; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java index 705c6eb..5722d8b 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -113,4 +114,14 @@ public class UDFOPPlus extends UDFBaseNumericOp { return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().add(b.getBigDecimal())); + return bigDecimalWritable; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java index c2c45e4..0711890 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -66,4 +67,9 @@ public class UDFOPPositive extends UDFBaseNumericUnaryOp { return a; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + return a; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java index 3b86e9c..4467a90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -108,4 +111,16 @@ public class UDFPosMod extends UDFBaseNumericOp { doubleWritable.set(((a.get() % b.get()) + b.get()) % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + BigDecimal av = a.getBigDecimal(); + BigDecimal bv = b.getBigDecimal(); + bigDecimalWritable.set(av.remainder(bv).add(bv).remainder(bv)); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java index 197adbb..9e211a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,8 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; extended = "Example:\n" + " > SELECT _FUNC_(2, 3) FROM src LIMIT 1;\n" + " 8") public class UDFPower extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable resultDouble = new DoubleWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public UDFPower() { } @@ -43,9 +45,8 @@ public class UDFPower extends UDF { if (a == null || b == null) { return null; } else { - result.set(Math.pow(a.get(), b.get())); - return result; + resultDouble.set(Math.pow(a.get(), b.get())); + return resultDouble; } } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java index 1841739..9957c15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java @@ -19,10 +19,12 @@ package org.apache.hadoop.hive.ql.udf; import java.math.BigDecimal; +import java.math.MathContext; import java.math.RoundingMode; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -36,8 +38,9 @@ import org.apache.hadoop.io.LongWritable; extended = "Example:\n" + " > SELECT _FUNC_(12.3456, 1) FROM src LIMIT 1;\n" + " 12.3'") public class UDFRound extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); - private LongWritable longWritable = new LongWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); + private final LongWritable longWritable = new LongWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public UDFRound() { } @@ -65,4 +68,14 @@ public class UDFRound extends UDF { return doubleWritable; } + public BigDecimalWritable evaluate(BigDecimalWritable n, IntWritable i) { + if ((n == null) || (i == null)) { + return null; + } + BigDecimal bd = n.getBigDecimal(); + MathContext mc = new MathContext( + bd.precision() - (bd.scale() - i.get()), RoundingMode.HALF_UP); + bigDecimalWritable.set(bd.round(mc)); + return bigDecimalWritable; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java index 6717e9d..67343ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -30,7 +31,7 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; extended = "Example:\n " + " > SELECT _FUNC_(4) FROM src LIMIT 1;\n" + " 2") public class UDFSqrt extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFSqrt() { } @@ -49,4 +50,17 @@ public class UDFSqrt extends UDF { } } + public DoubleWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + double v = i.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } else { + result.set(Math.sqrt(i.getBigDecimal().doubleValue())); + return result; + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index f3afd33..e7001ae 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +38,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToBoolean extends UDF { - private BooleanWritable booleanWritable = new BooleanWritable(); + private final BooleanWritable booleanWritable = new BooleanWritable(); public UDFToBoolean() { } @@ -172,4 +175,13 @@ public class UDFToBoolean extends UDF { } } + public BooleanWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + booleanWritable.set(!BigDecimal.ZERO.equals(i.getBigDecimal())); + return booleanWritable; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index 1b3b744..c5830ea 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToByte extends UDF { - private ByteWritable byteWritable = new ByteWritable(); + private final ByteWritable byteWritable = new ByteWritable(); public UDFToByte() { } @@ -181,4 +182,12 @@ public class UDFToByte extends UDF { } } + public ByteWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + byteWritable.set(i.getBigDecimal().byteValue()); + return byteWritable; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java index ce4660c..c57e31e 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -34,7 +35,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToDouble extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); public UDFToDouble() { } @@ -183,4 +184,12 @@ public class UDFToDouble extends UDF { } } + public DoubleWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + doubleWritable.set(i.getBigDecimal().doubleValue()); + return doubleWritable; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java index c6b197e..61591e9 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +36,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToFloat extends UDF { - private FloatWritable floatWritable = new FloatWritable(); + private final FloatWritable floatWritable = new FloatWritable(); public UDFToFloat() { } @@ -184,4 +185,13 @@ public class UDFToFloat extends UDF { } } + public FloatWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + floatWritable.set(i.getBigDecimal().floatValue()); + return floatWritable; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index 9b9d7df..018b3de 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToInteger extends UDF { - private IntWritable intWritable = new IntWritable(); + private final IntWritable intWritable = new IntWritable(); public UDFToInteger() { } @@ -188,4 +189,13 @@ public class UDFToInteger extends UDF { } } + public IntWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + intWritable.set(i.getBigDecimal().intValue()); + return intWritable; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index c7ea66d..426bc64 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ import org.apache.hadoop.io.Text; * */ public class UDFToLong extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); public UDFToLong() { } @@ -192,4 +193,13 @@ public class UDFToLong extends UDF { } } + public LongWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + longWritable.set(i.getBigDecimal().longValue()); + return longWritable; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index 558d405..5f42865 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -181,4 +182,13 @@ public class UDFToShort extends UDF { } } + public ShortWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + shortWritable.set(i.getBigDecimal().shortValue()); + return shortWritable; + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java index 4a38f8c..1d06eb3 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -141,10 +142,19 @@ public class UDFToString extends UDF { } } + public Text evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + t.set(i.toString()); + return t; + } + } + public Text evaluate (BytesWritable bw) { if (null == bw) { return null; -} + } t.set(bw.getBytes(),0,bw.getLength()); return t; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java index 43ee547..d5c8e14 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java @@ -102,6 +102,7 @@ public class GenericUDAFCorrelation extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -110,6 +111,7 @@ public class GenericUDAFCorrelation extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCorrelationEvaluator(); case STRING: case BOOLEAN: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java index fdcedfb..f7f24f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java @@ -93,6 +93,7 @@ public class GenericUDAFCovariance extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -101,6 +102,7 @@ public class GenericUDAFCovariance extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceEvaluator(); case STRING: case BOOLEAN: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java index ef3023e..ecf7151 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java @@ -67,6 +67,7 @@ public class GenericUDAFCovarianceSample extends GenericUDAFCovariance { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -75,6 +76,7 @@ public class GenericUDAFCovarianceSample extends GenericUDAFCovariance { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceSampleEvaluator(); case STRING: case BOOLEAN: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java index e0f81e0..b31f1da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java @@ -45,7 +45,7 @@ import org.apache.hadoop.util.StringUtils; /** * Computes an approximate histogram of a numerical column using a user-specified number of bins. - * + * * The output is an array of (x,y) pairs as Hive struct objects that represents the histogram's * bin centers and heights. */ @@ -72,7 +72,7 @@ public class GenericUDAFHistogramNumeric extends AbstractGenericUDAFResolver { throw new UDFArgumentTypeException(parameters.length - 1, "Please specify exactly two arguments."); } - + // validate the first parameter, which is the expression to compute over if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, @@ -87,6 +87,7 @@ public class GenericUDAFHistogramNumeric extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; case STRING: case BOOLEAN: @@ -170,7 +171,7 @@ public class GenericUDAFHistogramNumeric extends AbstractGenericUDAFResolver { @Override public Object terminatePartial(AggregationBuffer agg) throws HiveException { - // Return a single ArrayList where the first element is the number of histogram bins, + // Return a single ArrayList where the first element is the number of histogram bins, // and subsequent elements represent histogram (x,y) pairs. StdAgg myagg = (StdAgg) agg; return myagg.histogram.serialize(); @@ -233,7 +234,7 @@ public class GenericUDAFHistogramNumeric extends AbstractGenericUDAFResolver { } - // Aggregation buffer definition and manipulation methods + // Aggregation buffer definition and manipulation methods static class StdAgg implements AggregationBuffer { NumericHistogram histogram; // the histogram object }; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java index 4193a97..eab6e1e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java @@ -46,7 +46,7 @@ import org.apache.hadoop.util.StringUtils; /** * Computes an approximate percentile (quantile) from an approximate histogram, for very * large numbers of rows where the regular percentile() UDAF might run out of memory. - * + * * The input is a single double value or an array of double values representing the quantiles * requested. The output, corresponding to the input, is either an single double value or an * array of doubles that are the quantile values. @@ -59,7 +59,7 @@ import org.apache.hadoop.util.StringUtils; extended = "'expr' can be any numeric column, including doubles and floats, and 'pc' is " + "either a single double/float with a requested percentile, or an array of double/" + "float with multiple percentiles. If 'nb' is not specified, the default " + - "approximation is done with 10,000 histogram bins, which means that if there are " + + "approximation is done with 10,000 histogram bins, which means that if there are " + "10,000 or fewer unique values in 'expr', you can expect an exact result. The " + "percentile() function always computes an exact percentile and can run out of " + "memory if there are too many unique values in a column, which necessitates " + @@ -77,7 +77,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { throw new UDFArgumentTypeException(parameters.length - 1, "Please specify either two or three arguments."); } - + // Validate the first parameter, which is the expression to compute over. This should be a // numeric primitive type. if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { @@ -93,6 +93,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; default: throw new UDFArgumentTypeException(0, @@ -147,7 +148,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[1])) { throw new UDFArgumentTypeException(1, - "The second argument must be a constant, but " + parameters[1].getTypeName() + + "The second argument must be a constant, but " + parameters[1].getTypeName() + " was passed instead."); } @@ -172,7 +173,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[2])) { throw new UDFArgumentTypeException(2, - "The third argument must be a constant, but " + parameters[2].getTypeName() + + "The third argument must be a constant, but " + parameters[2].getTypeName() + " was passed instead."); } } @@ -184,7 +185,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { return new GenericUDAFSinglePercentileApproxEvaluator(); } } - + public static class GenericUDAFSinglePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -234,7 +235,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { } } - + public static class GenericUDAFMultiplePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -299,7 +300,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { - if(partial == null) { + if(partial == null) { return; } PercentileAggBuf myagg = (PercentileAggBuf) agg; @@ -316,7 +317,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { } // merge histograms - myagg.histogram.merge(partialHistogram); + myagg.histogram.merge(partialHistogram); } @Override @@ -382,7 +383,7 @@ public class GenericUDAFPercentileApprox extends AbstractGenericUDAFResolver { } else { result = new double[1]; result[0] = PrimitiveObjectInspectorUtils.getDouble( - quantileObj, + quantileObj, (PrimitiveObjectInspector)quantileOI); } for(int ii = 0; ii < result.length; ++ii) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java index 2a1a617..5d0aa50 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Compute the standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "std,stddev,stddev_pop", value = "_FUNC_(x) - Returns the standard deviation of a set of numbers") @@ -56,6 +56,7 @@ public class GenericUDAFStd extends GenericUDAFVariance { case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdEvaluator(); case BOOLEAN: default: @@ -68,7 +69,7 @@ public class GenericUDAFStd extends GenericUDAFVariance { /** * Compute the standard deviation by extending GenericUDAFVarianceEvaluator * and overriding the terminate() method of the evaluator. - * + * */ public static class GenericUDAFStdEvaluator extends GenericUDAFVarianceEvaluator { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java index d5791ed..cde947c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Compute the sample standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "stddev_samp", value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers") @@ -55,6 +55,7 @@ public class GenericUDAFStdSample extends GenericUDAFVariance { case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdSampleEvaluator(); case BOOLEAN: default: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 5a20f87..f80491d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.hive.ql.udf.generic; +import java.math.BigDecimal; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -66,6 +69,8 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { case DOUBLE: case STRING: return new GenericUDAFSumDouble(); + case DECIMAL: + return new GenericUDAFSumBigDecimal(); case BOOLEAN: default: throw new UDFArgumentTypeException(0, @@ -75,6 +80,89 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { } /** + * GenericUDAFSumBigDecimal. + * + */ + public static class GenericUDAFSumBigDecimal extends GenericUDAFEvaluator { + private PrimitiveObjectInspector inputOI; + private BigDecimalWritable result; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { + assert (parameters.length == 1); + super.init(m, parameters); + result = new BigDecimalWritable(BigDecimal.ZERO); + inputOI = (PrimitiveObjectInspector) parameters[0]; + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + /** class for storing double sum value. */ + static class SumBigDecimalAgg implements AggregationBuffer { + boolean empty; + BigDecimal sum; + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + SumBigDecimalAgg agg = new SumBigDecimalAgg(); + reset(agg); + return agg; + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg bdAgg = (SumBigDecimalAgg) agg; + bdAgg.empty = true; + bdAgg.sum = BigDecimal.ZERO; + } + + boolean warned = false; + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { + assert (parameters.length == 1); + try { + merge(agg, parameters[0]); + } catch (NumberFormatException e) { + if (!warned) { + warned = true; + LOG.warn(getClass().getSimpleName() + " " + + StringUtils.stringifyException(e)); + LOG + .warn(getClass().getSimpleName() + + " ignoring similar exceptions."); + } + } + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException { + return terminate(agg); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException { + if (partial != null) { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + myagg.empty = false; + myagg.sum = myagg.sum.add( + PrimitiveObjectInspectorUtils.getBigDecimal(partial, inputOI)); + } + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + if (myagg.empty) { + return null; + } + result.set(myagg.sum); + return result; + } + + } + + /** * GenericUDAFSumDouble. * */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java index 0b40d5c..7bba95c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java @@ -43,7 +43,7 @@ import org.apache.hadoop.util.StringUtils; /** * Compute the variance. This class is extended by: GenericUDAFVarianceSample * GenericUDAFStd GenericUDAFStdSample - * + * */ @Description(name = "variance,var_pop", value = "_FUNC_(x) - Returns the variance of a set of numbers") @@ -72,6 +72,7 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver { case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceEvaluator(); case BOOLEAN: default: @@ -85,18 +86,18 @@ public class GenericUDAFVariance extends AbstractGenericUDAFResolver { * Evaluate the variance using the algorithm described by Chan, Golub, and LeVeque in * "Algorithms for computing the sample variance: analysis and recommendations" * The American Statistician, 37 (1983) pp. 242--247. - * + * * variance = variance1 + variance2 + n/(m*(m+n)) * pow(((m/n)*t1 - t2),2) - * + * * where: - variance is sum[x-avg^2] (this is actually n times the variance) * and is updated at every step. - n is the count of elements in chunk1 - m is - * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = + * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = * sum of elements in chunk2. * * This algorithm was proven to be numerically stable by J.L. Barlow in * "Error analysis of a pairwise summation algorithm to compute sample variance" * Numer. Math, 58 (1991) pp. 583--590 - * + * */ public static class GenericUDAFVarianceEvaluator extends GenericUDAFEvaluator { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java index 65d860d..fa549e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Compute the sample variance by extending GenericUDAFVariance and overriding * the terminate() method of the evaluator. - * + * */ @Description(name = "var_samp", value = "_FUNC_(x) - Returns the sample variance of a set of numbers") @@ -56,6 +56,7 @@ public class GenericUDAFVarianceSample extends GenericUDAFVariance { case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceSampleEvaluator(); case BOOLEAN: default: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java new file mode 100644 index 0000000..d6776d1 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.BigDecimalConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +@Description(name = "decimal", value = "_FUNC_(a) - cast a to decimal") +public class GenericUDFToDecimal extends GenericUDF { + + private PrimitiveObjectInspector argumentOI; + private BigDecimalConverter bdConverter; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 1) { + throw new UDFArgumentLengthException( + "The function DECIMAL requires at least one argument, got " + + arguments.length); + } + try { + argumentOI = (PrimitiveObjectInspector) arguments[0]; + } catch (ClassCastException e) { + throw new UDFArgumentException( + "The function DECIMAL takes only primitive types"); + } + + bdConverter = new BigDecimalConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector); + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + if (o0 == null) { + return null; + } + + return bdConverter.convert(o0); + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + StringBuilder sb = new StringBuilder(); + sb.append("CAST( "); + sb.append(children[0]); + sb.append(" AS DECIMAL)"); + return sb.toString(); + } + +} diff --git a/ql/src/test/queries/clientpositive/decimal_1.q b/ql/src/test/queries/clientpositive/decimal_1.q new file mode 100644 index 0000000..063afa8 --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_1.q @@ -0,0 +1,17 @@ +drop table decimal_1; + +create table decimal_1 (t decimal); +alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + +insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1; +select cast(t as boolean) from decimal_1 limit 1; +select cast(t as tinyint) from decimal_1 limit 1; +select cast(t as smallint) from decimal_1 limit 1; +select cast(t as int) from decimal_1 limit 1; +select cast(t as bigint) from decimal_1 limit 1; +select cast(t as float) from decimal_1 limit 1; +select cast(t as double) from decimal_1 limit 1; +select cast(t as string) from decimal_1 limit 1; + +drop table decimal_1; diff --git a/ql/src/test/queries/clientpositive/decimal_2.q b/ql/src/test/queries/clientpositive/decimal_2.q new file mode 100644 index 0000000..f2d7cb8 --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_2.q @@ -0,0 +1,17 @@ +drop table decimal_2; + +create table decimal_2 (t decimal); +alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; + +insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1; +select cast(t as boolean) from decimal_2 limit 1; +select cast(t as tinyint) from decimal_2 limit 1; +select cast(t as smallint) from decimal_2 limit 1; +select cast(t as int) from decimal_2 limit 1; +select cast(t as bigint) from decimal_2 limit 1; +select cast(t as float) from decimal_2 limit 1; +select cast(t as double) from decimal_2 limit 1; +select cast(t as string) from decimal_2 limit 1; + +drop table decimal_2; diff --git a/ql/src/test/results/clientpositive/decimal_1.q.out b/ql/src/test/results/clientpositive/decimal_1.q.out new file mode 100644 index 0000000..4cf6e0a --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_1.q.out @@ -0,0 +1,117 @@ +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_1 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_1 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] diff --git a/ql/src/test/results/clientpositive/decimal_2.q.out b/ql/src/test/results/clientpositive/decimal_2.q.out new file mode 100644 index 0000000..284d9a5 --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_2.q.out @@ -0,0 +1,117 @@ +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_2 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_2 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index e906a3f..4954b29 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspect import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -279,6 +280,10 @@ public final class SerDeUtils { sb.append(txt.toString()); break; } + case DECIMAL: { + sb.append(((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + } default: throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java index df8166b..62bf975 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java @@ -1,119 +1,119 @@ -+/** -+ * Licensed to the Apache Software Foundation (ASF) under one -+ * or more contributor license agreements. See the NOTICE file -+ * distributed with this work for additional information -+ * regarding copyright ownership. The ASF licenses this file -+ * to you under the Apache License, Version 2.0 (the -+ * "License"); you may not use this file except in compliance -+ * with the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+package org.apache.hadoop.hive.serde2.io; -+ -+import java.io.DataInput; -+import java.io.DataOutput; -+import java.io.IOException; -+import java.math.BigDecimal; -+import java.math.BigInteger; -+ -+import org.apache.commons.logging.Log; -+import org.apache.commons.logging.LogFactory; -+import org.apache.hadoop.hive.serde2.ByteStream.Output; -+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; -+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; -+import org.apache.hadoop.io.WritableComparable; -+import org.apache.hadoop.io.WritableUtils; -+ -+public class BigDecimalWritable implements WritableComparable { -+ -+ static final private Log LOG = LogFactory.getLog(BigDecimalWritable.class); -+ -+ private byte[] internalStorage = new byte[0]; -+ private int scale; -+ -+ public BigDecimalWritable() { -+ } -+ -+ public BigDecimalWritable(byte[] bytes, int scale) { -+ set(bytes, scale); -+ } -+ -+ public BigDecimalWritable(BigDecimalWritable writable) { -+ set(writable.getBigDecimal()); -+ } -+ -+ public BigDecimalWritable(BigDecimal value) { -+ set(value); -+ } -+ -+ public void set(BigDecimal value) { -+ set(value.unscaledValue().toByteArray(), value.scale()); -+ } -+ -+ public void set(BigDecimalWritable writable) { -+ set(writable.getBigDecimal()); -+ } -+ -+ public void set(byte[] bytes, int scale) { -+ this.internalStorage = bytes; -+ this.scale = scale; -+ } -+ -+ private final VInt vInt = new VInt(); -+ -+ public void setFromBytes(byte[] bytes, int offset, int length) { -+ LazyBinaryUtils.readVInt(bytes, offset, vInt); -+ scale = vInt.value; -+ offset += vInt.length; -+ LazyBinaryUtils.readVInt(bytes, offset, vInt); -+ offset += vInt.length; -+ if (internalStorage.length != vInt.value) { -+ internalStorage = new byte[vInt.value]; -+ } -+ System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); -+ } -+ -+ public BigDecimal getBigDecimal() { -+ return new BigDecimal(new BigInteger(internalStorage), scale); -+ } -+ -+ @Override -+ public void readFields(DataInput in) throws IOException { -+ scale = WritableUtils.readVInt(in); -+ int byteArrayLen = WritableUtils.readVInt(in); -+ if (internalStorage.length != byteArrayLen) { -+ internalStorage = new byte[byteArrayLen]; -+ } -+ in.readFully(internalStorage); -+ } -+ -+ @Override -+ public void write(DataOutput out) throws IOException { -+ WritableUtils.writeVInt(out, scale); -+ WritableUtils.writeVInt(out, internalStorage.length); -+ out.write(internalStorage); -+ } -+ -+ @Override -+ public int compareTo(BigDecimalWritable that) { -+ return getBigDecimal().compareTo(that.getBigDecimal()); -+ } -+ -+ public void writeToByteStream(Output byteStream) { -+ LazyBinaryUtils.writeVInt(byteStream, scale); -+ LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); -+ byteStream.write(internalStorage, 0, internalStorage.length); -+ } -+ -+ @Override -+ public String toString() { -+ return getBigDecimal().toString(); -+ } -+} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.ByteStream.Output; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +public class BigDecimalWritable implements WritableComparable { + + static final private Log LOG = LogFactory.getLog(BigDecimalWritable.class); + + private byte[] internalStorage = new byte[0]; + private int scale; + + public BigDecimalWritable() { + } + + public BigDecimalWritable(byte[] bytes, int scale) { + set(bytes, scale); + } + + public BigDecimalWritable(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public BigDecimalWritable(BigDecimal value) { + set(value); + } + + public void set(BigDecimal value) { + set(value.unscaledValue().toByteArray(), value.scale()); + } + + public void set(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public void set(byte[] bytes, int scale) { + this.internalStorage = bytes; + this.scale = scale; + } + + private final VInt vInt = new VInt(); + + public void setFromBytes(byte[] bytes, int offset, int length) { + LazyBinaryUtils.readVInt(bytes, offset, vInt); + scale = vInt.value; + offset += vInt.length; + LazyBinaryUtils.readVInt(bytes, offset, vInt); + offset += vInt.length; + if (internalStorage.length != vInt.value) { + internalStorage = new byte[vInt.value]; + } + System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); + } + + public BigDecimal getBigDecimal() { + return new BigDecimal(new BigInteger(internalStorage), scale); + } + + @Override + public void readFields(DataInput in) throws IOException { + scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + if (internalStorage.length != byteArrayLen) { + internalStorage = new byte[byteArrayLen]; + } + in.readFully(internalStorage); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, scale); + WritableUtils.writeVInt(out, internalStorage.length); + out.write(internalStorage); + } + + @Override + public int compareTo(BigDecimalWritable that) { + return getBigDecimal().compareTo(that.getBigDecimal()); + } + + public void writeToByteStream(Output byteStream) { + LazyBinaryUtils.writeVInt(byteStream, scale); + LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); + byteStream.write(internalStorage, 0, internalStorage.length); + } + + @Override + public String toString() { + return getBigDecimal().toString(); + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java new file mode 100644 index 0000000..5d8a6a1 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazybinary; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; + +public class LazyBinaryBigDecimal extends + LazyBinaryPrimitive { + + LazyBinaryBigDecimal(WritableBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + LazyBinaryBigDecimal(LazyBinaryBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + data.setFromBytes(bytes.getData(), start, length); + } + +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java index 86f098f..3111cbc 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector; @@ -75,6 +76,8 @@ public final class LazyBinaryFactory { return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi); case BINARY: return new LazyBinaryBinary((WritableBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBinaryBigDecimal((WritableBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyBinaryObject for " + p); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index 807a52b..823e3e0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -382,6 +384,14 @@ public class LazyBinarySerDe implements SerDe { t.writeToByteStream(byteStream); return warnedOnceNullMapKey; } + + case DECIMAL: { + BigDecimalObjectInspector bdoi = (BigDecimalObjectInspector) poi; + BigDecimalWritable t = bdoi.getPrimitiveWritableObject(obj); + t.writeToByteStream(byteStream); + return warnedOnceNullMapKey; + } + default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index e024cb9..cac20ff 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -203,6 +203,14 @@ public final class LazyBinaryUtils { recordInfo.elementSize += (byte) WritableUtils.decodeVIntSize(bytes[offset+4]); } break; + case DECIMAL: + // using vint instead of 4 bytes + LazyBinaryUtils.readVInt(bytes, offset, vInt); + recordInfo.elementOffset = 0; + recordInfo.elementSize = vInt.length; + LazyBinaryUtils.readVInt(bytes, offset + vInt.length, vInt); + recordInfo.elementSize += vInt.length + vInt.value; + break; default: { throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 7f61344..0481aff 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import java.util.HashMap; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -69,6 +70,8 @@ public final class PrimitiveObjectInspectorFactory { new JavaTimestampObjectInspector(); public static final JavaBinaryObjectInspector javaByteArrayObjectInspector = new JavaBinaryObjectInspector(); + public static final JavaBigDecimalObjectInspector javaBigDecimalObjectInspector = + new JavaBigDecimalObjectInspector(); public static final WritableBooleanObjectInspector writableBooleanObjectInspector = new WritableBooleanObjectInspector(); @@ -92,6 +95,8 @@ public final class PrimitiveObjectInspectorFactory { new WritableTimestampObjectInspector(); public static final WritableBinaryObjectInspector writableBinaryObjectInspector = new WritableBinaryObjectInspector(); + public static final WritableBigDecimalObjectInspector writableBigDecimalObjectInspector = + new WritableBigDecimalObjectInspector(); private static HashMap cachedPrimitiveWritableInspectorCache = new HashMap(); @@ -118,6 +123,8 @@ public final class PrimitiveObjectInspectorFactory { writableTimestampObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BINARY, writableBinaryObjectInspector); + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DECIMAL, + writableBigDecimalObjectInspector); } private static HashMap cachedPrimitiveJavaInspectorCache = @@ -145,6 +152,8 @@ public final class PrimitiveObjectInspectorFactory { javaTimestampObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BINARY, javaByteArrayObjectInspector); + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DECIMAL, + javaBigDecimalObjectInspector); } /** @@ -191,6 +200,8 @@ public final class PrimitiveObjectInspectorFactory { return new WritableConstantStringObjectInspector((Text)value); case TIMESTAMP: return new WritableConstantTimestampObjectInspector((TimestampWritable)value); + case DECIMAL: + return new BigDecimalWritable((BigDecimalWritable)value); case BINARY: return new WritableConstantBinaryObjectInspector((BytesWritable)value); case VOID: diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java new file mode 100644 index 0000000..672b106 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +/** + * A WritableConstantBigDecimalObjectInspector is a WritableBigDecimalObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantBigDecimalObjectInspector extends WritableBigDecimalObjectInspector + implements ConstantObjectInspector { + + private final BigDecimalWritable value; + + WritableConstantBigDecimalObjectInspector(BigDecimalWritable value) { + this.value = value; + } + + @Override + public BigDecimalWritable getWritableConstantValue() { + return value; + } +} -- 1.7.3.4 From 6531ce1496b748a97134261021ef0975d0e01309 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Thu, 25 Oct 2012 23:59:35 -0700 Subject: [PATCH 3/4] HIVE-2693: Add the ObjectInspector updates for the BigDecimal type --- serde/if/serde.thrift | 3 +- serde/src/gen/thrift/gen-cpp/serde_constants.cpp | 3 + serde/src/gen/thrift/gen-cpp/serde_constants.h | 1 + .../org/apache/hadoop/hive/serde/Constants.java | 3 + .../gen/thrift/gen-php/serde/serde_constants.php | 3 + .../org_apache_hadoop_hive_serde/constants.py | 2 + serde/src/gen/thrift/gen-rb/serde_constants.rb | 3 + .../objectinspector/PrimitiveObjectInspector.java | 2 +- .../primitive/BigDecimalObjectInspector.java | 33 ++++++++ .../primitive/JavaBigDecimalObjectInspector.java | 68 ++++++++++++++++ .../PrimitiveObjectInspectorConverter.java | 22 +++++ .../primitive/PrimitiveObjectInspectorFactory.java | 2 +- .../primitive/PrimitiveObjectInspectorUtils.java | 81 ++++++++++++++++++++ .../SettableBigDecimalObjectInspector.java | 39 ++++++++++ .../WritableBigDecimalObjectInspector.java | 75 ++++++++++++++++++ 15 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java diff --git a/serde/if/serde.thrift b/serde/if/serde.thrift index e40c697..9847720 100644 --- a/serde/if/serde.thrift +++ b/serde/if/serde.thrift @@ -53,6 +53,7 @@ const string STRING_TYPE_NAME = "string"; const string DATE_TYPE_NAME = "date"; const string DATETIME_TYPE_NAME = "datetime"; const string TIMESTAMP_TYPE_NAME = "timestamp"; +const string DECIMAL_TYPE_NAME = "decimal"; const string BINARY_TYPE_NAME = "binary"; const string LIST_TYPE_NAME = "array"; @@ -63,7 +64,7 @@ const string UNION_TYPE_NAME = "uniontype"; const string LIST_COLUMNS = "columns"; const string LIST_COLUMN_TYPES = "columns.types"; -const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME BINARY_TYPE_NAME], +const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME], const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], diff --git a/serde/src/gen/thrift/gen-cpp/serde_constants.cpp b/serde/src/gen/thrift/gen-cpp/serde_constants.cpp index 350521d..70ed38b 100644 --- a/serde/src/gen/thrift/gen-cpp/serde_constants.cpp +++ b/serde/src/gen/thrift/gen-cpp/serde_constants.cpp @@ -62,6 +62,8 @@ serdeConstants::serdeConstants() { TIMESTAMP_TYPE_NAME = "timestamp"; + DECIMAL_TYPE_NAME = "decimal"; + BINARY_TYPE_NAME = "binary"; LIST_TYPE_NAME = "array"; @@ -88,6 +90,7 @@ serdeConstants::serdeConstants() { PrimitiveTypes.insert("date"); PrimitiveTypes.insert("datetime"); PrimitiveTypes.insert("timestamp"); + PrimitiveTypes.insert("decimal"); PrimitiveTypes.insert("binary"); CollectionTypes.insert("array"); diff --git a/serde/src/gen/thrift/gen-cpp/serde_constants.h b/serde/src/gen/thrift/gen-cpp/serde_constants.h index 94015c2..06d092e 100644 --- a/serde/src/gen/thrift/gen-cpp/serde_constants.h +++ b/serde/src/gen/thrift/gen-cpp/serde_constants.h @@ -40,6 +40,7 @@ class serdeConstants { std::string DATE_TYPE_NAME; std::string DATETIME_TYPE_NAME; std::string TIMESTAMP_TYPE_NAME; + std::string DECIMAL_TYPE_NAME; std::string BINARY_TYPE_NAME; std::string LIST_TYPE_NAME; std::string MAP_TYPE_NAME; diff --git a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java index 2c32999..79d1942 100644 --- a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java +++ b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/Constants.java @@ -75,6 +75,8 @@ public class Constants { public static final String TIMESTAMP_TYPE_NAME = "timestamp"; + public static final String DECIMAL_TYPE_NAME = "decimal"; + public static final String BINARY_TYPE_NAME = "binary"; public static final String LIST_TYPE_NAME = "array"; @@ -103,6 +105,7 @@ public class Constants { PrimitiveTypes.add("date"); PrimitiveTypes.add("datetime"); PrimitiveTypes.add("timestamp"); + PrimitiveTypes.add("decimal"); PrimitiveTypes.add("binary"); } diff --git a/serde/src/gen/thrift/gen-php/serde/serde_constants.php b/serde/src/gen/thrift/gen-php/serde/serde_constants.php index 99bda06..d93b6ca 100644 --- a/serde/src/gen/thrift/gen-php/serde/serde_constants.php +++ b/serde/src/gen/thrift/gen-php/serde/serde_constants.php @@ -60,6 +60,8 @@ $GLOBALS['serde_CONSTANTS']['DATETIME_TYPE_NAME'] = "datetime"; $GLOBALS['serde_CONSTANTS']['TIMESTAMP_TYPE_NAME'] = "timestamp"; +$GLOBALS['serde_CONSTANTS']['DECIMAL_TYPE_NAME'] = "decimal"; + $GLOBALS['serde_CONSTANTS']['BINARY_TYPE_NAME'] = "binary"; $GLOBALS['serde_CONSTANTS']['LIST_TYPE_NAME'] = "array"; @@ -87,6 +89,7 @@ $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array( "date" => true, "datetime" => true, "timestamp" => true, + "decimal" => true, "binary" => true, ); diff --git a/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py b/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py index b9e01c1..db20e31 100644 --- a/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py +++ b/serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py @@ -33,6 +33,7 @@ STRING_TYPE_NAME = "string" DATE_TYPE_NAME = "date" DATETIME_TYPE_NAME = "datetime" TIMESTAMP_TYPE_NAME = "timestamp" +DECIMAL_TYPE_NAME = "decimal" BINARY_TYPE_NAME = "binary" LIST_TYPE_NAME = "array" MAP_TYPE_NAME = "map" @@ -53,6 +54,7 @@ PrimitiveTypes = set([ "date", "datetime", "timestamp", + "decimal", "binary", ]) CollectionTypes = set([ diff --git a/serde/src/gen/thrift/gen-rb/serde_constants.rb b/serde/src/gen/thrift/gen-rb/serde_constants.rb index 7ef2fc9..2c6684e 100644 --- a/serde/src/gen/thrift/gen-rb/serde_constants.rb +++ b/serde/src/gen/thrift/gen-rb/serde_constants.rb @@ -58,6 +58,8 @@ DATETIME_TYPE_NAME = %q"datetime" TIMESTAMP_TYPE_NAME = %q"timestamp" +DECIMAL_TYPE_NAME = %q"decimal" + BINARY_TYPE_NAME = %q"binary" LIST_TYPE_NAME = %q"array" @@ -85,6 +87,7 @@ PrimitiveTypes = Set.new([ %q"date", %q"datetime", %q"timestamp", + %q"decimal", %q"binary", ]) diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java index 970e884..aace3bb 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -27,7 +27,7 @@ public interface PrimitiveObjectInspector extends ObjectInspector { * The primitive types supported by Hive. */ public static enum PrimitiveCategory { - VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, UNKNOWN + VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, UNKNOWN }; /** diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java new file mode 100644 index 0000000..44db243 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +/** + * A DecimalObjectInspector inspects an Object representing a BigDecimal. + */ +public interface BigDecimalObjectInspector extends PrimitiveObjectInspector { + + BigDecimalWritable getPrimitiveWritableObject(Object o); + + BigDecimal getPrimitiveJavaObject(Object o); +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java new file mode 100644 index 0000000..382d7e8 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class JavaBigDecimalObjectInspector + extends AbstractPrimitiveJavaObjectInspector + implements SettableBigDecimalObjectInspector { + + protected JavaBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : (BigDecimal) o; + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object set(Object o, BigDecimal t) { + return t; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + return t == null ? null : t.getBigDecimal(); + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object create(BigDecimal t) { + return t == null ? null : new BigDecimal(t.unscaledValue(), t.scale()); + } + +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index f0b16fa..e74034c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; +import java.math.BigDecimal; import java.sql.Timestamp; import org.apache.hadoop.hive.serde2.ByteStream; @@ -257,6 +258,27 @@ public class PrimitiveObjectInspectorConverter { } } + public static class BigDecimalConverter implements Converter { + + PrimitiveObjectInspector inputOI; + SettableBigDecimalObjectInspector outputOI; + Object r; + + public BigDecimalConverter(PrimitiveObjectInspector inputOI, + SettableBigDecimalObjectInspector outputOI) { + this.inputOI = inputOI; + this.outputOI = outputOI; + this.r = outputOI.create(BigDecimal.ZERO); + } + + @Override + public Object convert(Object input) { + return outputOI.set(r, PrimitiveObjectInspectorUtils.getBigDecimal(input, + inputOI)); + } + + } + public static class BinaryConverter implements Converter{ PrimitiveObjectInspector inputOI; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 0481aff..a39934c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -201,7 +201,7 @@ public final class PrimitiveObjectInspectorFactory { case TIMESTAMP: return new WritableConstantTimestampObjectInspector((TimestampWritable)value); case DECIMAL: - return new BigDecimalWritable((BigDecimalWritable)value); + return new WritableConstantBigDecimalObjectInspector((BigDecimalWritable)value); case BINARY: return new WritableConstantBinaryObjectInspector((BytesWritable)value); case VOID: diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 5430814..3f384a1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -21,11 +21,13 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.math.BigDecimal; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -178,6 +180,9 @@ public final class PrimitiveObjectInspectorUtils { public static final PrimitiveTypeEntry timestampTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.TIMESTAMP, Constants.TIMESTAMP_TYPE_NAME, null, Object.class, TimestampWritable.class); + public static final PrimitiveTypeEntry decimalTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.DECIMAL, Constants.DECIMAL_TYPE_NAME, null, + Object.class, BigDecimalWritable.class); // The following is a complex type for special handling public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( @@ -195,6 +200,7 @@ public final class PrimitiveObjectInspectorUtils { registerType(byteTypeEntry); registerType(shortTypeEntry); registerType(timestampTypeEntry); + registerType(decimalTypeEntry); registerType(unknownTypeEntry); } @@ -362,6 +368,10 @@ public final class PrimitiveObjectInspectorUtils { return ((BinaryObjectInspector) oi1).getPrimitiveWritableObject(o1). equals(((BinaryObjectInspector) oi2).getPrimitiveWritableObject(o2)); } + case DECIMAL: { + return ((BigDecimalObjectInspector) oi1).getPrimitiveJavaObject(o1) + .equals(((BigDecimalObjectInspector) oi2).getPrimitiveJavaObject(o2)); + } default: return false; } @@ -391,6 +401,8 @@ public final class PrimitiveObjectInspectorUtils { case TIMESTAMP: return ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); + case DECIMAL: + return ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o).doubleValue(); default: throw new NumberFormatException(); } @@ -465,6 +477,10 @@ public final class PrimitiveObjectInspectorUtils { result = (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds() != 0); break; + case DECIMAL: + result = BigDecimal.ZERO.equals( + ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -545,6 +561,10 @@ public final class PrimitiveObjectInspectorUtils { result = (int) (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds()); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).intValue(); + break; default: { throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -599,6 +619,10 @@ public final class PrimitiveObjectInspectorUtils { result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).longValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -646,6 +670,10 @@ public final class PrimitiveObjectInspectorUtils { case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getDouble(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).doubleValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -706,6 +734,10 @@ public final class PrimitiveObjectInspectorUtils { case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).toString(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -739,6 +771,55 @@ public final class PrimitiveObjectInspectorUtils { } } + public static BigDecimal getBigDecimal(Object o, PrimitiveObjectInspector oi) { + if (o == null) { + return null; + } + + BigDecimal result = null; + switch (oi.getPrimitiveCategory()) { + case VOID: + result = null; + break; + case BOOLEAN: + result = ((BooleanObjectInspector) oi).get(o) ? + BigDecimal.ONE : BigDecimal.ZERO; + break; + case BYTE: + result = new BigDecimal(((ByteObjectInspector) oi).get(o)); + break; + case SHORT: + result = new BigDecimal(((ShortObjectInspector) oi).get(o)); + break; + case INT: + result = new BigDecimal(((IntObjectInspector) oi).get(o)); + break; + case LONG: + result = new BigDecimal(((LongObjectInspector) oi).get(o)); + break; + case FLOAT: + result = new BigDecimal(((FloatObjectInspector) oi).get(o)); + break; + case DOUBLE: + result = new BigDecimal(((DoubleObjectInspector) oi).get(o)); + break; + case STRING: + result = new BigDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + case TIMESTAMP: + result = new BigDecimal(((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) + .getDouble()); + break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + break; + default: + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + return result; + } + public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { if (o == null) { return null; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java new file mode 100644 index 0000000..ff262b2 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +/** + * A SettableDecimalObjectInspector can set a BigDecimal value to an object. + */ +public interface SettableBigDecimalObjectInspector extends BigDecimalObjectInspector { + + Object set(Object o, byte[] bytes, int scale); + + Object set(Object o, BigDecimal t); + + Object set(Object o, BigDecimalWritable t); + + Object create(byte[] bytes, int scale); + + Object create (BigDecimal t); + +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java new file mode 100644 index 0000000..88184cf --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class WritableBigDecimalObjectInspector + extends AbstractPrimitiveWritableObjectInspector + implements SettableBigDecimalObjectInspector { + + protected WritableBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : (BigDecimalWritable) o; + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((BigDecimalWritable) o).getBigDecimal(); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimalWritable) o); + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + ((BigDecimalWritable) o).set(bytes, scale); + return o; + } + + @Override + public Object set(Object o, BigDecimal t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimalWritable(bytes, scale); + } + + @Override + public Object create(BigDecimal t) { + return new BigDecimalWritable(t); + } + +} -- 1.7.3.4 From bf0ab2f6acebc86bcf40b9c7978ad28f365f7462 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Fri, 26 Oct 2012 15:44:36 -0700 Subject: [PATCH 4/4] HIVE-2693: Wire up objectinspectors and lazy configs for the decimal type --- .../hadoop/hive/serde2/io/BigDecimalWritable.java | 14 +++++ .../hadoop/hive/serde2/lazy/LazyBigDecimal.java | 54 ++++++++++++++++++++ .../hadoop/hive/serde2/lazy/LazyFactory.java | 5 ++- .../apache/hadoop/hive/serde2/lazy/LazyUtils.java | 8 +++ .../primitive/LazyBigDecimalObjectInspector.java | 28 ++++++++++ .../LazyPrimitiveObjectInspectorFactory.java | 4 ++ .../objectinspector/ObjectInspectorConverters.java | 5 ++ .../objectinspector/ObjectInspectorUtils.java | 12 ++++ .../hive/serde2/typeinfo/TypeInfoFactory.java | 1 + 9 files changed, 130 insertions(+), 1 deletions(-) create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java index 62bf975..c2e1194 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java @@ -116,4 +116,18 @@ public class BigDecimalWritable implements WritableComparable { + static final private Log LOG = LogFactory.getLog(LazyBigDecimal.class); + + public LazyBigDecimal(LazyBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + public LazyBigDecimal(LazyBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + /** + * Initilizes LazyBigDecimal object by interpreting the input bytes + * as a numeric string + * + * @param bytes + * @param start + * @param length + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; + try { + byteData = Text.decode(bytes.getData(), start, length); + data.set(new BigDecimal(byteData)); + isNull = false; + } catch (NumberFormatException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null. Given data is :" + + byteData, e); + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null.", e); + } + } + + @Override + public BigDecimalWritable getWritableObject() { + return data; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 20758a7..2c6251f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -26,8 +26,9 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector; @@ -112,6 +113,8 @@ public final class LazyFactory { return new LazyTimestamp((LazyTimestampObjectInspector) oi); case BINARY: return new LazyBinary((LazyBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBigDecimal((LazyBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyObject for " + p); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java index bcdb3d3..5bfeba4 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -235,6 +237,12 @@ public final class LazyUtils { ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break; } + case DECIMAL: { + BigDecimal bd = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + ByteBuffer b = Text.encode(bd.toString()); + out.write(b.array(), 0, b.limit()); + break; + } default: { throw new RuntimeException("Hive internal error."); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java new file mode 100644 index 0000000..15eb0f6 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.LazyBigDecimal; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public class LazyBigDecimalObjectInspector + extends AbstractPrimitiveLazyObjectInspector + implements BigDecimalObjectInspector { + + protected LazyBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new LazyBigDecimal((LazyBigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((LazyBigDecimal) o).getWritableObject().getBigDecimal(); + } + +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index 57d2fad..1e0ad00 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -57,6 +57,8 @@ public final class LazyPrimitiveObjectInspectorFactory { new LazyTimestampObjectInspector(); public static final LazyBinaryObjectInspector LAZY_BINARY_OBJECT_INSPECTOR = new LazyBinaryObjectInspector(); + public static final LazyBigDecimalObjectInspector LAZY_BIG_DECIMAL_OBJECT_INSPECTOR = + new LazyBigDecimalObjectInspector(); static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>(); @@ -101,6 +103,8 @@ public final class LazyPrimitiveObjectInspectorFactory { return LAZY_VOID_OBJECT_INSPECTOR; case TIMESTAMP: return LAZY_TIMESTAMP_OBJECT_INSPECTOR; + case DECIMAL: + return LAZY_BIG_DECIMAL_OBJECT_INSPECTOR; default: throw new RuntimeException("Internal error: Cannot find ObjectInspector " + " for " + primitiveCategory); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java index 7537e99..6e1aac0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; @@ -117,6 +118,10 @@ public final class ObjectInspectorConverters { return new PrimitiveObjectInspectorConverter.BinaryConverter( (PrimitiveObjectInspector)inputOI, (SettableBinaryObjectInspector)outputOI); + case DECIMAL: + return new PrimitiveObjectInspectorConverter.BigDecimalConverter( + (PrimitiveObjectInspector) inputOI, + (SettableBigDecimalObjectInspector) outputOI); default: throw new RuntimeException("Hive internal error: conversion of " diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index bedc9dd..cbd97e9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -30,10 +30,12 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -470,6 +472,9 @@ public final class ObjectInspectorUtils { TimestampWritable t = ((TimestampObjectInspector) poi) .getPrimitiveWritableObject(o); return t.hashCode(); + case DECIMAL: + return ((BigDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); + default: { throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory()); @@ -654,6 +659,13 @@ public final class ObjectInspectorUtils { .getPrimitiveWritableObject(o2); return t1.compareTo(t2); } + case DECIMAL: { + BigDecimalWritable t1 = ((BigDecimalObjectInspector) poi1) + .getPrimitiveWritableObject(o1); + BigDecimalWritable t2 = ((BigDecimalObjectInspector) poi2) + .getPrimitiveWritableObject(o2); + return t1.compareTo(t2); + } default: { throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 594f5ab..f40e90d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -64,6 +64,7 @@ public final class TypeInfoFactory { public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(Constants.SMALLINT_TYPE_NAME); public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(Constants.TIMESTAMP_TYPE_NAME); public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(Constants.BINARY_TYPE_NAME); + public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(Constants.DECIMAL_TYPE_NAME); public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); -- 1.7.3.4