Index: build.properties =================================================================== --- build.properties (revision 1409060) +++ build.properties (working copy) @@ -79,7 +79,7 @@ # (measured in milliseconds). Ignored if fork is disabled. When running # multiple tests inside the same Java VM (see forkMode), timeout # applies to the time that all tests use together, not to an individual test. -test.junit.timeout=36400000 +test.junit.timeout=364000000 # Use this property to selectively disable tests from the command line: # ant test -Dtest.junit.exclude="**/TestCliDriver.class" Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1409060) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -391,7 +391,7 @@ org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME, "string"); typeToThriftTypeMap.put( org.apache.hadoop.hive.serde.serdeConstants.BINARY_TYPE_NAME, "binary"); - // These 3 types are not supported yet. + // These 4 types are not supported yet. // We should define a complex type date in thrift that contains a single int // member, and DynamicSerDe // should convert it to date type at runtime. @@ -402,6 +402,8 @@ typeToThriftTypeMap .put(org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME, "timestamp"); + typeToThriftTypeMap.put( + org.apache.hadoop.hive.serde.serdeConstants.DECIMAL_TYPE_NAME, "decimal"); } /** Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBigDecimal.java (working copy) @@ -0,0 +1,54 @@ +package org.apache.hadoop.hive.serde2.lazy; + +import java.math.BigDecimal; +import java.nio.charset.CharacterCodingException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBigDecimalObjectInspector; +import org.apache.hadoop.io.Text; + +public class LazyBigDecimal extends LazyPrimitive { + static final private Log LOG = LogFactory.getLog(LazyBigDecimal.class); + + public LazyBigDecimal(LazyBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + public LazyBigDecimal(LazyBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + /** + * Initilizes LazyBigDecimal object by interpreting the input bytes + * as a numeric string + * + * @param bytes + * @param start + * @param length + */ + @Override + public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; + try { + byteData = Text.decode(bytes.getData(), start, length); + data.set(new BigDecimal(byteData)); + isNull = false; + } catch (NumberFormatException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null. Given data is :" + + byteData, e); + } catch (CharacterCodingException e) { + isNull = true; + LOG.debug("Data not in the BigDecimal data type range so converted to null.", e); + } + } + + @Override + public BigDecimalWritable getWritableObject() { + return data; + } +} Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (working copy) @@ -21,6 +21,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -235,6 +237,12 @@ ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break; } + case DECIMAL: { + BigDecimal bd = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + ByteBuffer b = Text.encode(bd.toString()); + out.write(b.array(), 0, b.limit()); + break; + } default: { throw new RuntimeException("Hive internal error."); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (working copy) @@ -26,8 +26,9 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBigDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector; @@ -112,6 +113,8 @@ return new LazyTimestamp((LazyTimestampObjectInspector) oi); case BINARY: return new LazyBinary((LazyBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBigDecimal((LazyBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyObject for " + p); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (working copy) @@ -57,6 +57,8 @@ new LazyTimestampObjectInspector(); public static final LazyBinaryObjectInspector LAZY_BINARY_OBJECT_INSPECTOR = new LazyBinaryObjectInspector(); + public static final LazyBigDecimalObjectInspector LAZY_BIG_DECIMAL_OBJECT_INSPECTOR = + new LazyBigDecimalObjectInspector(); static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>(); @@ -101,6 +103,8 @@ return LAZY_VOID_OBJECT_INSPECTOR; case TIMESTAMP: return LAZY_TIMESTAMP_OBJECT_INSPECTOR; + case DECIMAL: + return LAZY_BIG_DECIMAL_OBJECT_INSPECTOR; default: throw new RuntimeException("Internal error: Cannot find ObjectInspector " + " for " + primitiveCategory); Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBigDecimalObjectInspector.java (working copy) @@ -0,0 +1,28 @@ +package org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.LazyBigDecimal; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public class LazyBigDecimalObjectInspector + extends AbstractPrimitiveLazyObjectInspector + implements BigDecimalObjectInspector { + + protected LazyBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new LazyBigDecimal((LazyBigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((LazyBigDecimal) o).getWritableObject().getBigDecimal(); + } + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (working copy) @@ -203,6 +203,14 @@ recordInfo.elementSize += (byte) WritableUtils.decodeVIntSize(bytes[offset+4]); } break; + case DECIMAL: + // using vint instead of 4 bytes + LazyBinaryUtils.readVInt(bytes, offset, vInt); + recordInfo.elementOffset = 0; + recordInfo.elementSize = vInt.length; + LazyBinaryUtils.readVInt(bytes, offset + vInt.length, vInt); + recordInfo.elementSize += vInt.length + vInt.value; + break; default: { throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory); Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (working copy) @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector; @@ -75,6 +76,8 @@ return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi); case BINARY: return new LazyBinaryBinary((WritableBinaryObjectInspector) oi); + case DECIMAL: + return new LazyBinaryBigDecimal((WritableBigDecimalObjectInspector) oi); default: throw new RuntimeException("Internal error: no LazyBinaryObject for " + p); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBigDecimal.java (working copy) @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.lazybinary; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBigDecimalObjectInspector; + +public class LazyBinaryBigDecimal extends + LazyBinaryPrimitive { + + LazyBinaryBigDecimal(WritableBigDecimalObjectInspector oi) { + super(oi); + data = new BigDecimalWritable(); + } + + LazyBinaryBigDecimal(LazyBinaryBigDecimal copy) { + super(copy); + data = new BigDecimalWritable(copy.data); + } + + @Override + public void init(ByteArrayRef bytes, int start, int length) { + data.setFromBytes(bytes.getData(), start, length); + } + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java (working copy) @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -382,6 +384,14 @@ t.writeToByteStream(byteStream); return warnedOnceNullMapKey; } + + case DECIMAL: { + BigDecimalObjectInspector bdoi = (BigDecimalObjectInspector) poi; + BigDecimalWritable t = bdoi.getPrimitiveWritableObject(obj); + t.writeToByteStream(byteStream); + return warnedOnceNullMapKey; + } + default: { throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (working copy) @@ -64,6 +64,7 @@ public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); + public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(serdeConstants.DECIMAL_TYPE_NAME); public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (working copy) @@ -30,10 +30,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -470,6 +472,9 @@ TimestampWritable t = ((TimestampObjectInspector) poi) .getPrimitiveWritableObject(o); return t.hashCode(); + case DECIMAL: + return ((BigDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); + default: { throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory()); @@ -654,6 +659,13 @@ .getPrimitiveWritableObject(o2); return t1.compareTo(t2); } + case DECIMAL: { + BigDecimalWritable t1 = ((BigDecimalObjectInspector) poi1) + .getPrimitiveWritableObject(o1); + BigDecimalWritable t2 = ((BigDecimalObjectInspector) poi2) + .getPrimitiveWritableObject(o2); + return t1.compareTo(t2); + } default: { throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory()); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (working copy) @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; @@ -117,6 +118,10 @@ return new PrimitiveObjectInspectorConverter.BinaryConverter( (PrimitiveObjectInspector)inputOI, (SettableBinaryObjectInspector)outputOI); + case DECIMAL: + return new PrimitiveObjectInspectorConverter.BigDecimalConverter( + (PrimitiveObjectInspector) inputOI, + (SettableBigDecimalObjectInspector) outputOI); default: throw new RuntimeException("Hive internal error: conversion of " Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java (working copy) @@ -27,7 +27,7 @@ * The primitive types supported by Hive. */ public static enum PrimitiveCategory { - VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, UNKNOWN + VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP, BINARY, DECIMAL, UNKNOWN }; /** Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBigDecimalObjectInspector.java (working copy) @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class WritableBigDecimalObjectInspector + extends AbstractPrimitiveWritableObjectInspector + implements SettableBigDecimalObjectInspector { + + protected WritableBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : (BigDecimalWritable) o; + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : ((BigDecimalWritable) o).getBigDecimal(); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimalWritable) o); + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + ((BigDecimalWritable) o).set(bytes, scale); + return o; + } + + @Override + public Object set(Object o, BigDecimal t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + ((BigDecimalWritable) o).set(t); + return o; + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimalWritable(bytes, scale); + } + + @Override + public Object create(BigDecimal t) { + return new BigDecimalWritable(t); + } + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBigDecimalObjectInspector.java (working copy) @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +/** + * A SettableDecimalObjectInspector can set a BigDecimal value to an object. + */ +public interface SettableBigDecimalObjectInspector extends BigDecimalObjectInspector { + + Object set(Object o, byte[] bytes, int scale); + + Object set(Object o, BigDecimal t); + + Object set(Object o, BigDecimalWritable t); + + Object create(byte[] bytes, int scale); + + Object create (BigDecimal t); + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (working copy) @@ -21,11 +21,13 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.math.BigDecimal; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -178,6 +180,9 @@ public static final PrimitiveTypeEntry timestampTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME, null, Object.class, TimestampWritable.class); + public static final PrimitiveTypeEntry decimalTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.DECIMAL, serdeConstants.DECIMAL_TYPE_NAME, null, + Object.class, BigDecimalWritable.class); // The following is a complex type for special handling public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( @@ -195,6 +200,7 @@ registerType(byteTypeEntry); registerType(shortTypeEntry); registerType(timestampTypeEntry); + registerType(decimalTypeEntry); registerType(unknownTypeEntry); } @@ -362,6 +368,10 @@ return ((BinaryObjectInspector) oi1).getPrimitiveWritableObject(o1). equals(((BinaryObjectInspector) oi2).getPrimitiveWritableObject(o2)); } + case DECIMAL: { + return ((BigDecimalObjectInspector) oi1).getPrimitiveJavaObject(o1) + .equals(((BigDecimalObjectInspector) oi2).getPrimitiveJavaObject(o2)); + } default: return false; } @@ -391,6 +401,8 @@ case TIMESTAMP: return ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getDouble(); + case DECIMAL: + return ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o).doubleValue(); default: throw new NumberFormatException(); } @@ -465,6 +477,10 @@ result = (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds() != 0); break; + case DECIMAL: + result = BigDecimal.ZERO.equals( + ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -545,6 +561,10 @@ result = (int) (((TimestampObjectInspector) oi) .getPrimitiveWritableObject(o).getSeconds()); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).intValue(); + break; default: { throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -599,6 +619,10 @@ result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) .getSeconds(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).longValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -646,6 +670,10 @@ case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getDouble(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).doubleValue(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -706,6 +734,10 @@ case TIMESTAMP: result = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).toString(); break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o).toString(); + break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " + oi.getTypeName()); @@ -739,6 +771,55 @@ } } + public static BigDecimal getBigDecimal(Object o, PrimitiveObjectInspector oi) { + if (o == null) { + return null; + } + + BigDecimal result = null; + switch (oi.getPrimitiveCategory()) { + case VOID: + result = null; + break; + case BOOLEAN: + result = ((BooleanObjectInspector) oi).get(o) ? + BigDecimal.ONE : BigDecimal.ZERO; + break; + case BYTE: + result = new BigDecimal(((ByteObjectInspector) oi).get(o)); + break; + case SHORT: + result = new BigDecimal(((ShortObjectInspector) oi).get(o)); + break; + case INT: + result = new BigDecimal(((IntObjectInspector) oi).get(o)); + break; + case LONG: + result = new BigDecimal(((LongObjectInspector) oi).get(o)); + break; + case FLOAT: + result = new BigDecimal(((FloatObjectInspector) oi).get(o)); + break; + case DOUBLE: + result = new BigDecimal(((DoubleObjectInspector) oi).get(o)); + break; + case STRING: + result = new BigDecimal(((StringObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + case TIMESTAMP: + result = new BigDecimal(((TimestampObjectInspector) oi).getPrimitiveWritableObject(o) + .getDouble()); + break; + case DECIMAL: + result = ((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + break; + default: + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + return result; + } + public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { if (o == null) { return null; Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java (working copy) @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; +import java.math.BigDecimal; import java.sql.Timestamp; import org.apache.hadoop.hive.serde2.ByteStream; @@ -257,6 +258,27 @@ } } + public static class BigDecimalConverter implements Converter { + + PrimitiveObjectInspector inputOI; + SettableBigDecimalObjectInspector outputOI; + Object r; + + public BigDecimalConverter(PrimitiveObjectInspector inputOI, + SettableBigDecimalObjectInspector outputOI) { + this.inputOI = inputOI; + this.outputOI = outputOI; + this.r = outputOI.create(BigDecimal.ZERO); + } + + @Override + public Object convert(Object input) { + return outputOI.set(r, PrimitiveObjectInspectorUtils.getBigDecimal(input, + inputOI)); + } + + } + public static class BinaryConverter implements Converter{ PrimitiveObjectInspector inputOI; Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBigDecimalObjectInspector.java (working copy) @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; + +public class JavaBigDecimalObjectInspector + extends AbstractPrimitiveJavaObjectInspector + implements SettableBigDecimalObjectInspector { + + protected JavaBigDecimalObjectInspector() { + super(PrimitiveObjectInspectorUtils.decimalTypeEntry); + } + + @Override + public BigDecimalWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new BigDecimalWritable((BigDecimal) o); + } + + @Override + public BigDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : (BigDecimal) o; + } + + @Override + public Object set(Object o, byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object set(Object o, BigDecimal t) { + return t; + } + + @Override + public Object set(Object o, BigDecimalWritable t) { + return t == null ? null : t.getBigDecimal(); + } + + @Override + public Object create(byte[] bytes, int scale) { + return new BigDecimal(new BigInteger(bytes), scale); + } + + @Override + public Object create(BigDecimal t) { + return t == null ? null : new BigDecimal(t.unscaledValue(), t.scale()); + } + +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBigDecimalObjectInspector.java (working copy) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +/** + * A WritableConstantBigDecimalObjectInspector is a WritableBigDecimalObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantBigDecimalObjectInspector extends WritableBigDecimalObjectInspector + implements ConstantObjectInspector { + + private final BigDecimalWritable value; + + WritableConstantBigDecimalObjectInspector(BigDecimalWritable value) { + this.value = value; + } + + @Override + public BigDecimalWritable getWritableConstantValue() { + return value; + } +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java (working copy) @@ -20,6 +20,7 @@ import java.util.HashMap; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -69,6 +70,8 @@ new JavaTimestampObjectInspector(); public static final JavaBinaryObjectInspector javaByteArrayObjectInspector = new JavaBinaryObjectInspector(); + public static final JavaBigDecimalObjectInspector javaBigDecimalObjectInspector = + new JavaBigDecimalObjectInspector(); public static final WritableBooleanObjectInspector writableBooleanObjectInspector = new WritableBooleanObjectInspector(); @@ -92,6 +95,8 @@ new WritableTimestampObjectInspector(); public static final WritableBinaryObjectInspector writableBinaryObjectInspector = new WritableBinaryObjectInspector(); + public static final WritableBigDecimalObjectInspector writableBigDecimalObjectInspector = + new WritableBigDecimalObjectInspector(); private static HashMap cachedPrimitiveWritableInspectorCache = new HashMap(); @@ -118,6 +123,8 @@ writableTimestampObjectInspector); cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BINARY, writableBinaryObjectInspector); + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DECIMAL, + writableBigDecimalObjectInspector); } private static HashMap cachedPrimitiveJavaInspectorCache = @@ -145,6 +152,8 @@ javaTimestampObjectInspector); cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BINARY, javaByteArrayObjectInspector); + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DECIMAL, + javaBigDecimalObjectInspector); } /** @@ -191,6 +200,8 @@ return new WritableConstantStringObjectInspector((Text)value); case TIMESTAMP: return new WritableConstantTimestampObjectInspector((TimestampWritable)value); + case DECIMAL: + return new WritableConstantBigDecimalObjectInspector((BigDecimalWritable)value); case BINARY: return new WritableConstantBinaryObjectInspector((BytesWritable)value); case VOID: Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BigDecimalObjectInspector.java (working copy) @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import java.math.BigDecimal; + +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + +/** + * A DecimalObjectInspector inspects an Object representing a BigDecimal. + */ +public interface BigDecimalObjectInspector extends PrimitiveObjectInspector { + + BigDecimalWritable getPrimitiveWritableObject(Object o); + + BigDecimal getPrimitiveJavaObject(Object o); +} Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (revision 1409060) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (working copy) @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BigDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; @@ -279,6 +280,10 @@ sb.append(txt.toString()); break; } + case DECIMAL: { + sb.append(((BigDecimalObjectInspector) oi).getPrimitiveJavaObject(o)); + break; + } default: throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); Index: serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/io/BigDecimalWritable.java (working copy) @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.serde2.ByteStream.Output; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +public class BigDecimalWritable implements WritableComparable { + + static final private Log LOG = LogFactory.getLog(BigDecimalWritable.class); + + private byte[] internalStorage = new byte[0]; + private int scale; + + public BigDecimalWritable() { + } + + public BigDecimalWritable(byte[] bytes, int scale) { + set(bytes, scale); + } + + public BigDecimalWritable(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public BigDecimalWritable(BigDecimal value) { + set(value); + } + + public void set(BigDecimal value) { + set(value.unscaledValue().toByteArray(), value.scale()); + } + + public void set(BigDecimalWritable writable) { + set(writable.getBigDecimal()); + } + + public void set(byte[] bytes, int scale) { + this.internalStorage = bytes; + this.scale = scale; + } + + private final VInt vInt = new VInt(); + + public void setFromBytes(byte[] bytes, int offset, int length) { + LazyBinaryUtils.readVInt(bytes, offset, vInt); + scale = vInt.value; + offset += vInt.length; + LazyBinaryUtils.readVInt(bytes, offset, vInt); + offset += vInt.length; + if (internalStorage.length != vInt.value) { + internalStorage = new byte[vInt.value]; + } + System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); + } + + public BigDecimal getBigDecimal() { + return new BigDecimal(new BigInteger(internalStorage), scale); + } + + @Override + public void readFields(DataInput in) throws IOException { + scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + if (internalStorage.length != byteArrayLen) { + internalStorage = new byte[byteArrayLen]; + } + in.readFully(internalStorage); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, scale); + WritableUtils.writeVInt(out, internalStorage.length); + out.write(internalStorage); + } + + @Override + public int compareTo(BigDecimalWritable that) { + return getBigDecimal().compareTo(that.getBigDecimal()); + } + + public void writeToByteStream(Output byteStream) { + LazyBinaryUtils.writeVInt(byteStream, scale); + LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); + byteStream.write(internalStorage, 0, internalStorage.length); + } + + @Override + public String toString() { + return getBigDecimal().toString(); + } + + @Override + public boolean equals(Object other) { + if (other == null || !(other instanceof BigDecimalWritable)) { + return false; + } + BigDecimalWritable bdw = (BigDecimalWritable) other; + return getBigDecimal().equals(bdw.getBigDecimal()); + } + + @Override + public int hashCode() { + return getBigDecimal().hashCode(); + } +} Index: serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py =================================================================== --- serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py (revision 1409060) +++ serde/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py (working copy) @@ -35,6 +35,7 @@ DATE_TYPE_NAME = "date" DATETIME_TYPE_NAME = "datetime" TIMESTAMP_TYPE_NAME = "timestamp" +DECIMAL_TYPE_NAME = "decimal" BINARY_TYPE_NAME = "binary" LIST_TYPE_NAME = "array" MAP_TYPE_NAME = "map" @@ -55,6 +56,7 @@ "date", "datetime", "timestamp", + "decimal", "binary", ]) CollectionTypes = set([ Index: serde/src/gen/thrift/gen-cpp/serde_constants.cpp =================================================================== --- serde/src/gen/thrift/gen-cpp/serde_constants.cpp (revision 1409060) +++ serde/src/gen/thrift/gen-cpp/serde_constants.cpp (working copy) @@ -63,6 +63,8 @@ TIMESTAMP_TYPE_NAME = "timestamp"; + DECIMAL_TYPE_NAME = "decimal"; + BINARY_TYPE_NAME = "binary"; LIST_TYPE_NAME = "array"; @@ -89,6 +91,7 @@ PrimitiveTypes.insert("date"); PrimitiveTypes.insert("datetime"); PrimitiveTypes.insert("timestamp"); + PrimitiveTypes.insert("decimal"); PrimitiveTypes.insert("binary"); CollectionTypes.insert("array"); Index: serde/src/gen/thrift/gen-cpp/serde_constants.h =================================================================== --- serde/src/gen/thrift/gen-cpp/serde_constants.h (revision 1409060) +++ serde/src/gen/thrift/gen-cpp/serde_constants.h (working copy) @@ -41,6 +41,7 @@ std::string DATE_TYPE_NAME; std::string DATETIME_TYPE_NAME; std::string TIMESTAMP_TYPE_NAME; + std::string DECIMAL_TYPE_NAME; std::string BINARY_TYPE_NAME; std::string LIST_TYPE_NAME; std::string MAP_TYPE_NAME; Index: serde/src/gen/thrift/gen-rb/serde_constants.rb =================================================================== --- serde/src/gen/thrift/gen-rb/serde_constants.rb (revision 1409060) +++ serde/src/gen/thrift/gen-rb/serde_constants.rb (working copy) @@ -59,6 +59,8 @@ TIMESTAMP_TYPE_NAME = %q"timestamp" +DECIMAL_TYPE_NAME = %q"decimal" + BINARY_TYPE_NAME = %q"binary" LIST_TYPE_NAME = %q"array" @@ -86,6 +88,7 @@ %q"date", %q"datetime", %q"timestamp", + %q"decimal", %q"binary", ]) Index: serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php =================================================================== --- serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php (revision 1409060) +++ serde/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php (working copy) @@ -68,6 +68,8 @@ $GLOBALS['serde_CONSTANTS']['TIMESTAMP_TYPE_NAME'] = "timestamp"; +$GLOBALS['serde_CONSTANTS']['DECIMAL_TYPE_NAME'] = "decimal"; + $GLOBALS['serde_CONSTANTS']['BINARY_TYPE_NAME'] = "binary"; $GLOBALS['serde_CONSTANTS']['LIST_TYPE_NAME'] = "array"; @@ -95,6 +97,7 @@ "date" => true, "datetime" => true, "timestamp" => true, + "decimal" => true, "binary" => true, ); Index: serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java =================================================================== --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java (revision 1409060) +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java (working copy) @@ -85,6 +85,8 @@ public static final String TIMESTAMP_TYPE_NAME = "timestamp"; + public static final String DECIMAL_TYPE_NAME = "decimal"; + public static final String BINARY_TYPE_NAME = "binary"; public static final String LIST_TYPE_NAME = "array"; @@ -113,6 +115,7 @@ PrimitiveTypes.add("date"); PrimitiveTypes.add("datetime"); PrimitiveTypes.add("timestamp"); + PrimitiveTypes.add("decimal"); PrimitiveTypes.add("binary"); } Index: serde/if/serde.thrift =================================================================== --- serde/if/serde.thrift (revision 1409060) +++ serde/if/serde.thrift (working copy) @@ -53,6 +53,7 @@ const string DATE_TYPE_NAME = "date"; const string DATETIME_TYPE_NAME = "datetime"; const string TIMESTAMP_TYPE_NAME = "timestamp"; +const string DECIMAL_TYPE_NAME = "decimal"; const string BINARY_TYPE_NAME = "binary"; const string LIST_TYPE_NAME = "array"; @@ -63,7 +64,7 @@ const string LIST_COLUMNS = "columns"; const string LIST_COLUMN_TYPES = "columns.types"; -const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME BINARY_TYPE_NAME], +const set PrimitiveTypes = [ VOID_TYPE_NAME BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME DECIMAL_TYPE_NAME BINARY_TYPE_NAME], const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], Index: ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out =================================================================== --- ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out (revision 1409060) +++ ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out (working copy) @@ -1011,7 +1011,7 @@ expr: _col4 type: int expr: ((_col5 - _col2) / _col2) - type: double + type: decimal outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false Index: ql/src/test/results/clientpositive/decimal_2.q.out =================================================================== --- ql/src/test/results/clientpositive/decimal_2.q.out (revision 0) +++ ql/src/test/results/clientpositive/decimal_2.q.out (working copy) @@ -0,0 +1,117 @@ +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_2 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_2 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +PREHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_2 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_2 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_2 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] +17.29 +PREHOOK: query: drop table decimal_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_2 +PREHOOK: Output: default@decimal_2 +POSTHOOK: query: drop table decimal_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_2 +POSTHOOK: Output: default@decimal_2 +POSTHOOK: Lineage: decimal_2.t EXPRESSION [] Index: ql/src/test/results/clientpositive/decimal_1.q.out =================================================================== --- ql/src/test/results/clientpositive/decimal_1.q.out (revision 0) +++ ql/src/test/results/clientpositive/decimal_1.q.out (working copy) @@ -0,0 +1,117 @@ +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table decimal_1 (t decimal) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table decimal_1 (t decimal) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +PREHOOK: type: ALTERTABLE_SERIALIZER +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +POSTHOOK: type: ALTERTABLE_SERIALIZER +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +PREHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +PREHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as boolean) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +true +PREHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as tinyint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as smallint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as int) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as int) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as bigint) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17 +PREHOOK: query: select cast(t as float) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as float) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as double) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as double) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: select cast(t as string) from decimal_1 limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(t as string) from decimal_1 limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_1 +#### A masked pattern was here #### +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] +17.29 +PREHOOK: query: drop table decimal_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 +POSTHOOK: query: drop table decimal_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 +POSTHOOK: Lineage: decimal_1.t EXPRESSION [] Index: ql/src/test/queries/clientpositive/decimal_1.q =================================================================== --- ql/src/test/queries/clientpositive/decimal_1.q (revision 0) +++ ql/src/test/queries/clientpositive/decimal_1.q (working copy) @@ -0,0 +1,17 @@ +drop table decimal_1; + +create table decimal_1 (t decimal); +alter table decimal_1 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + +insert overwrite table decimal_1 + select cast('17.29' as decimal) from src limit 1; +select cast(t as boolean) from decimal_1 limit 1; +select cast(t as tinyint) from decimal_1 limit 1; +select cast(t as smallint) from decimal_1 limit 1; +select cast(t as int) from decimal_1 limit 1; +select cast(t as bigint) from decimal_1 limit 1; +select cast(t as float) from decimal_1 limit 1; +select cast(t as double) from decimal_1 limit 1; +select cast(t as string) from decimal_1 limit 1; + +drop table decimal_1; Index: ql/src/test/queries/clientpositive/decimal_2.q =================================================================== --- ql/src/test/queries/clientpositive/decimal_2.q (revision 0) +++ ql/src/test/queries/clientpositive/decimal_2.q (working copy) @@ -0,0 +1,17 @@ +drop table decimal_2; + +create table decimal_2 (t decimal); +alter table decimal_2 set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; + +insert overwrite table decimal_2 + select cast('17.29' as decimal) from src limit 1; +select cast(t as boolean) from decimal_2 limit 1; +select cast(t as tinyint) from decimal_2 limit 1; +select cast(t as smallint) from decimal_2 limit 1; +select cast(t as int) from decimal_2 limit 1; +select cast(t as bigint) from decimal_2 limit 1; +select cast(t as float) from decimal_2 limit 1; +select cast(t as double) from decimal_2 limit 1; +select cast(t as string) from decimal_2 limit 1; + +drop table decimal_2; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -199,6 +199,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTranslate; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion; @@ -401,6 +402,8 @@ GenericUDFTimestamp.class); registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, GenericUDFToBinary.class); + registerGenericUDF(serdeConstants.DECIMAL_TYPE_NAME, + GenericUDFToDecimal.class); // Aggregate functions registerGenericUDAF("max", new GenericUDAFMax()); @@ -634,7 +637,8 @@ registerNumericType(serdeConstants.BIGINT_TYPE_NAME, 4); registerNumericType(serdeConstants.FLOAT_TYPE_NAME, 5); registerNumericType(serdeConstants.DOUBLE_TYPE_NAME, 6); - registerNumericType(serdeConstants.STRING_TYPE_NAME, 7); + registerNumericType(serdeConstants.DECIMAL_TYPE_NAME, 7); + registerNumericType(serdeConstants.STRING_TYPE_NAME, 8); } /** @@ -703,27 +707,23 @@ return (ai > bi) ? a : b; } - /** - * Returns whether it is possible to implicitly convert an object of Class - * from to Class to. - */ - public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { + public static int implicitConvertableCost(TypeInfo from, TypeInfo to) { if (from.equals(to)) { - return true; + return 1; } // Allow implicit String to Double conversion if (from.equals(TypeInfoFactory.stringTypeInfo) && to.equals(TypeInfoFactory.doubleTypeInfo)) { - return true; + return 1; } // Void can be converted to any type if (from.equals(TypeInfoFactory.voidTypeInfo)) { - return true; + return 1; } if (from.equals(TypeInfoFactory.timestampTypeInfo) && to.equals(TypeInfoFactory.stringTypeInfo)) { - return true; + return 1; } // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double @@ -731,12 +731,28 @@ Integer f = numericTypes.get(from); Integer t = numericTypes.get(to); if (f == null || t == null) { - return false; + return -1; } if (f.intValue() > t.intValue()) { + return -1; + } + return ( + numericTypes.get(TypeInfoFactory.stringTypeInfo).intValue() - t.intValue()); + } + + + /** + * Returns whether it is possible to implicitly convert an object of Class + * from to Class to. + */ + public static boolean implicitConvertable(TypeInfo from, TypeInfo to) { + int cost = implicitConvertableCost(from, to); + + if (cost < 0) { return false; + } else { + return true; } - return true; } /** @@ -896,11 +912,13 @@ * (usually 0 for no conversion and 1 for conversion). */ public static int matchCost(TypeInfo argumentPassed, - TypeInfo argumentAccepted, boolean exact) { + TypeInfo argumentAccepted, boolean exact, boolean expanded) + throws UDFArgumentException { if (argumentAccepted.equals(argumentPassed)) { // matches return 0; } + if (argumentPassed.equals(TypeInfoFactory.voidTypeInfo)) { // passing null matches everything return 0; @@ -912,7 +930,7 @@ .getListElementTypeInfo(); TypeInfo argumentAcceptedElement = ((ListTypeInfo) argumentAccepted) .getListElementTypeInfo(); - return matchCost(argumentPassedElement, argumentAcceptedElement, exact); + return matchCost(argumentPassedElement, argumentAcceptedElement, exact, expanded); } if (argumentPassed.getCategory().equals(Category.MAP) && argumentAccepted.getCategory().equals(Category.MAP)) { @@ -925,8 +943,9 @@ .getMapValueTypeInfo(); TypeInfo argumentAcceptedValue = ((MapTypeInfo) argumentAccepted) .getMapValueTypeInfo(); - int cost1 = matchCost(argumentPassedKey, argumentAcceptedKey, exact); - int cost2 = matchCost(argumentPassedValue, argumentAcceptedValue, exact); + int cost1 = matchCost(argumentPassedKey, argumentAcceptedKey, exact, expanded); + int cost2 = matchCost(argumentPassedValue, argumentAcceptedValue, exact, expanded); + if (cost1 < 0 || cost2 < 0) { return -1; } @@ -938,13 +957,43 @@ // but there is a conversion cost. return 1; } - if (!exact && implicitConvertable(argumentPassed, argumentAccepted)) { + if (!exact && !expanded && implicitConvertable(argumentPassed, argumentAccepted)) { return 1; + } else if (!exact && expanded) { + return implicitConvertableCost(argumentPassed, argumentAccepted); } return -1; } + /* + public static enum ConversionPrecedence { + BIGDECIMAL(1, "bigdecimal"), + DOUBLE(2, "double"), + FLOAT(3, "float"), + LONG(4, "long"), + INTEGER(5, "int"), + BOOLEAN(6, "boolean"), + ; + + public final String typeName; + public final int typePrecedence; + + ConversionPrecedence(int precedence, String name) { + this.typeName = name; + this.typePrecedence = precedence; + } + + String getTypeName() { + return this.typeName; + } + + int findPrecedence(String name) { + return this.typePrecedence; + } + } + */ + /** * Gets the closest matching method corresponding to the argument list from a * list of methods. @@ -964,6 +1013,7 @@ List udfMethods = new ArrayList(); // The cost of the result int leastConversionCost = Integer.MAX_VALUE; + boolean expanded = false; for (Method m : mlist) { List argumentsAccepted = TypeInfoUtils.getParameterTypeInfos(m, @@ -978,7 +1028,7 @@ for (int i = 0; i < argumentsPassed.size() && match; i++) { int cost = matchCost(argumentsPassed.get(i), argumentsAccepted.get(i), - exact); + exact, expanded); if (cost == -1) { match = false; } else { @@ -1017,7 +1067,28 @@ } if (udfMethods.size() > 1) { // Ambiguous method found - throw new AmbiguousMethodException(udfClass, argumentsPassed, mlist); + Method lowestCostMethod = null; + expanded = true; + for (Method m : udfMethods) { + int conversionCost = 0; + int lowestCost = Integer.MAX_VALUE; + List argumentsAccepted = + TypeInfoUtils.getParameterTypeInfos(m, argumentsPassed.size()); + for (int i = 0; i < argumentsPassed.size(); i++) { + int cost = 0; + cost = matchCost(argumentsPassed.get(i), argumentsAccepted.get(i), exact, expanded); + if (cost == -1) { + break; + } + conversionCost += cost; + } + if (conversionCost < lowestCost) { + lowestCost = conversionCost; + lowestCostMethod = m; + } + } + + return lowestCostMethod; } return udfMethods.get(0); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (working copy) @@ -501,6 +501,8 @@ serdeConstants.BINARY_TYPE_NAME); conversionFunctionTextHashMap.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME); + conversionFunctionTextHashMap.put(HiveParser.TOK_DECIMAL, + serdeConstants.DECIMAL_TYPE_NAME); } public static boolean isRedundantConversionFunction(ASTNode expr, Index: ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -106,6 +106,7 @@ TOK_TIMESTAMP; TOK_STRING; TOK_BINARY; +TOK_DECIMAL; TOK_LIST; TOK_STRUCT; TOK_MAP; @@ -1451,6 +1452,7 @@ | KW_TIMESTAMP -> TOK_TIMESTAMP | KW_STRING -> TOK_STRING | KW_BINARY -> TOK_BINARY + | KW_DECIMAL -> TOK_DECIMAL ; listType @@ -2387,6 +2389,7 @@ KW_DATE: 'DATE'; KW_DATETIME: 'DATETIME'; KW_TIMESTAMP: 'TIMESTAMP'; +KW_DECIMAL: 'DECIMAL'; KW_STRING: 'STRING'; KW_ARRAY: 'ARRAY'; KW_STRUCT: 'STRUCT'; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -151,6 +151,7 @@ TokenToTypeName.put(HiveParser.TOK_DATE, serdeConstants.DATE_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_DATETIME, serdeConstants.DATETIME_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_TIMESTAMP, serdeConstants.TIMESTAMP_TYPE_NAME); + TokenToTypeName.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); } public static String getTypeName(int token) throws SemanticException { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToLong extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); public UDFToLong() { } @@ -192,4 +193,13 @@ } } + public LongWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + longWritable.set(i.getBigDecimal().longValue()); + return longWritable; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -33,7 +34,7 @@ public class UDFLog2 extends UDF { private static double log2 = Math.log(2.0); - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog2() { } @@ -50,4 +51,17 @@ } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v) / log2); + return result; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToByte extends UDF { - private ByteWritable byteWritable = new ByteWritable(); + private final ByteWritable byteWritable = new ByteWritable(); public UDFToByte() { } @@ -181,4 +182,12 @@ } } + public ByteWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + byteWritable.set(i.getBigDecimal().byteValue()); + return byteWritable; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -34,7 +35,7 @@ * */ public class UDFToDouble extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); public UDFToDouble() { } @@ -183,4 +184,12 @@ } } + public DoubleWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + doubleWritable.set(i.getBigDecimal().doubleValue()); + return doubleWritable; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMultiply.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,14 @@ doubleWritable.set(a.get() * b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().multiply(b.getBigDecimal())); + return bigDecimalWritable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericOp.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.NumericOpMethodResolver; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -48,6 +49,7 @@ protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a, ByteWritable b); @@ -61,4 +63,5 @@ public abstract DoubleWritable evaluate(DoubleWritable a, DoubleWritable b); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,8 @@ extended = "Example:\n" + " > SELECT _FUNC_(2, 3) FROM src LIMIT 1;\n" + " 8") public class UDFPower extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable resultDouble = new DoubleWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public UDFPower() { } @@ -43,9 +45,8 @@ if (a == null || b == null) { return null; } else { - result.set(Math.pow(a.get(), b.get())); - return result; + resultDouble.set(Math.pow(a.get(), b.get())); + return resultDouble; } } - } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPosMod.java (working copy) @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -108,4 +111,16 @@ doubleWritable.set(((a.get() % b.get()) + b.get()) % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + BigDecimal av = a.getBigDecimal(); + BigDecimal bv = b.getBigDecimal(); + bigDecimalWritable.set(av.remainder(bv).add(bv).remainder(bv)); + return bigDecimalWritable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPDivide.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -29,12 +30,13 @@ @Description(name = "/", value = "a _FUNC_ b - Divide a by b", extended = "Example:\n" + " > SELECT 3 _FUNC_ 2 FROM src LIMIT 1;\n" + " 1.5") /** - * Note that in SQL, the return type of divide is not necessarily the same + * Note that in SQL, the return type of divide is not necessarily the same * as the parameters. For example, 3 / 2 = 1.5, not 1. To follow SQL, we always * return a double for divide. */ public class UDFOPDivide extends UDF { - private DoubleWritable doubleWritable = new DoubleWritable(); + private final DoubleWritable doubleWritable = new DoubleWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public DoubleWritable evaluate(DoubleWritable a, DoubleWritable b) { // LOG.info("Get input " + a.getClass() + ":" + a + " " + b.getClass() + ":" @@ -46,4 +48,13 @@ doubleWritable.set(a.get() / b.get()); return doubleWritable; } + + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().divide(b.getBigDecimal())); + return bigDecimalWritable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java (working copy) @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ + " 0\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFCeil extends UDF { - private LongWritable longWritable = new LongWritable(); + private final LongWritable longWritable = new LongWritable(); + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public UDFCeil() { } @@ -48,4 +54,15 @@ } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + MathContext mc = new MathContext( + bd.precision() - bd.scale(), RoundingMode.CEILING); + bigDecimalWritable.set(bd.round(mc)); + return bigDecimalWritable; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java (working copy) @@ -18,7 +18,10 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; + import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +38,7 @@ * */ public class UDFToBoolean extends UDF { - private BooleanWritable booleanWritable = new BooleanWritable(); + private final BooleanWritable booleanWritable = new BooleanWritable(); public UDFToBoolean() { } @@ -172,4 +175,13 @@ } } + public BooleanWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + booleanWritable.set(!BigDecimal.ZERO.equals(i.getBigDecimal())); + return booleanWritable; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -181,4 +182,13 @@ } } + public ShortWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + shortWritable.set(i.getBigDecimal().shortValue()); + return shortWritable; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java (working copy) @@ -18,8 +18,13 @@ package org.apache.hadoop.hive.ql.udf; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; @@ -34,7 +39,8 @@ + " -1\n" + " > SELECT _FUNC_(5) FROM src LIMIT 1;\n" + " 5") public class UDFFloor extends UDF { - private LongWritable result = new LongWritable(); + private final LongWritable result = new LongWritable(); + private final BigDecimalWritable bdResult = new BigDecimalWritable(); public UDFFloor() { } @@ -48,4 +54,15 @@ } } + public BigDecimalWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + BigDecimal bd = i.getBigDecimal(); + MathContext mc = new MathContext( + bd.precision() - bd.scale(), RoundingMode.FLOOR); + bdResult.set(bd.round(mc)); + return bdResult; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -141,10 +142,19 @@ } } + public Text evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + t.set(i.toString()); + return t; + } + } + public Text evaluate (BytesWritable bw) { if (null == bw) { return null; -} + } t.set(bw.getBytes(),0,bw.getLength()); return t; } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,7 @@ extended = "Example:\n" + " > SELECT _FUNC_(13, 13) FROM src LIMIT 1;\n" + " 1") public class UDFLog extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog() { } @@ -48,6 +49,19 @@ } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v)); + return result; + } + } + /** * Returns the logarithm of "a" with base "base". */ @@ -60,4 +74,19 @@ } } + /** + * Returns the logarithm of "a" with base "base". + */ + public DoubleWritable evaluate(DoubleWritable base, BigDecimalWritable a) { + if (a == null || base == null || base.get() <= 1.0) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v <= 0.0) { + return null; + } + result.set(Math.log(v) / Math.log(base.get())); + return result; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMinus.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,15 @@ doubleWritable.set(a.get() - b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().subtract(b.getBigDecimal())); + return bigDecimalWritable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -36,7 +37,7 @@ * */ public class UDFToInteger extends UDF { - private IntWritable intWritable = new IntWritable(); + private final IntWritable intWritable = new IntWritable(); public UDFToInteger() { } @@ -188,4 +189,13 @@ } } + public IntWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + intWritable.set(i.getBigDecimal().intValue()); + return intWritable; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -66,4 +67,9 @@ return a; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + return a; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFAbs.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -38,6 +39,7 @@ private final DoubleWritable resultDouble = new DoubleWritable(); private final LongWritable resultLong = new LongWritable(); private final IntWritable resultInt = new IntWritable(); + private final BigDecimalWritable resultBigDecimal = new BigDecimalWritable(); public DoubleWritable evaluate(DoubleWritable n) { if (n == null) { @@ -68,4 +70,13 @@ return resultInt; } + + public BigDecimalWritable evaluate(BigDecimalWritable n) { + if (n == null) { + return null; + } + + resultBigDecimal.set(n.getBigDecimal().abs()); + return resultBigDecimal; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRound.java (working copy) @@ -19,10 +19,12 @@ package org.apache.hadoop.hive.ql.udf; import java.math.BigDecimal; +import java.math.MathContext; import java.math.RoundingMode; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.IntWritable; @@ -35,6 +37,7 @@ extended = "Example:\n" + " > SELECT _FUNC_(12.3456, 1) FROM src LIMIT 1;\n" + " 12.3'") public class UDFRound extends UDF { + private final BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); private final DoubleWritable doubleWritable = new DoubleWritable(); public UDFRound() { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSqrt.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -30,7 +31,7 @@ extended = "Example:\n " + " > SELECT _FUNC_(4) FROM src LIMIT 1;\n" + " 2") public class UDFSqrt extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFSqrt() { } @@ -49,4 +50,17 @@ } } + public DoubleWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + double v = i.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } else { + result.set(Math.sqrt(i.getBigDecimal().doubleValue())); + return result; + } + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -90,4 +91,13 @@ return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } + bigDecimalWritable.set(a.getBigDecimal().negate()); + return bigDecimalWritable; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -45,6 +46,7 @@ protected LongWritable longWritable = new LongWritable(); protected FloatWritable floatWritable = new FloatWritable(); protected DoubleWritable doubleWritable = new DoubleWritable(); + protected BigDecimalWritable bigDecimalWritable = new BigDecimalWritable(); public abstract ByteWritable evaluate(ByteWritable a); @@ -58,4 +60,5 @@ public abstract DoubleWritable evaluate(DoubleWritable a); + public abstract BigDecimalWritable evaluate(BigDecimalWritable a); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPMod.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -107,4 +108,14 @@ doubleWritable.set(a.get() % b.get()); return doubleWritable; } + + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().remainder(b.getBigDecimal())); + return bigDecimalWritable; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -31,7 +32,7 @@ extended = "Example:\n " + " > SELECT _FUNC_(0) FROM src LIMIT 1;\n" + " 1") public class UDFExp extends UDF { - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFExp() { } @@ -48,4 +49,12 @@ } } + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + result.set(Math.exp(a.getBigDecimal().doubleValue())); + return result; + } + } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java (working copy) @@ -28,7 +28,7 @@ /** * Compute the standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "std,stddev,stddev_pop", value = "_FUNC_(x) - Returns the standard deviation of a set of numbers") @@ -56,6 +56,7 @@ case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdEvaluator(); case BOOLEAN: default: @@ -68,7 +69,7 @@ /** * Compute the standard deviation by extending GenericUDAFVarianceEvaluator * and overriding the terminate() method of the evaluator. - * + * */ public static class GenericUDAFStdEvaluator extends GenericUDAFVarianceEvaluator { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java (working copy) @@ -46,7 +46,7 @@ /** * Computes an approximate percentile (quantile) from an approximate histogram, for very * large numbers of rows where the regular percentile() UDAF might run out of memory. - * + * * The input is a single double value or an array of double values representing the quantiles * requested. The output, corresponding to the input, is either an single double value or an * array of doubles that are the quantile values. @@ -59,7 +59,7 @@ extended = "'expr' can be any numeric column, including doubles and floats, and 'pc' is " + "either a single double/float with a requested percentile, or an array of double/" + "float with multiple percentiles. If 'nb' is not specified, the default " + - "approximation is done with 10,000 histogram bins, which means that if there are " + + "approximation is done with 10,000 histogram bins, which means that if there are " + "10,000 or fewer unique values in 'expr', you can expect an exact result. The " + "percentile() function always computes an exact percentile and can run out of " + "memory if there are too many unique values in a column, which necessitates " + @@ -77,7 +77,7 @@ throw new UDFArgumentTypeException(parameters.length - 1, "Please specify either two or three arguments."); } - + // Validate the first parameter, which is the expression to compute over. This should be a // numeric primitive type. if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { @@ -93,6 +93,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; default: throw new UDFArgumentTypeException(0, @@ -147,7 +148,7 @@ // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[1])) { throw new UDFArgumentTypeException(1, - "The second argument must be a constant, but " + parameters[1].getTypeName() + + "The second argument must be a constant, but " + parameters[1].getTypeName() + " was passed instead."); } @@ -172,7 +173,7 @@ // Also make sure it is a constant. if (!ObjectInspectorUtils.isConstantObjectInspector(parameters[2])) { throw new UDFArgumentTypeException(2, - "The third argument must be a constant, but " + parameters[2].getTypeName() + + "The third argument must be a constant, but " + parameters[2].getTypeName() + " was passed instead."); } } @@ -184,7 +185,7 @@ return new GenericUDAFSinglePercentileApproxEvaluator(); } } - + public static class GenericUDAFSinglePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -234,7 +235,7 @@ } } - + public static class GenericUDAFMultiplePercentileApproxEvaluator extends GenericUDAFPercentileApproxEvaluator { @@ -299,7 +300,7 @@ @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { - if(partial == null) { + if(partial == null) { return; } PercentileAggBuf myagg = (PercentileAggBuf) agg; @@ -316,7 +317,7 @@ } // merge histograms - myagg.histogram.merge(partialHistogram); + myagg.histogram.merge(partialHistogram); } @Override @@ -382,7 +383,7 @@ } else { result = new double[1]; result[0] = PrimitiveObjectInspectorUtils.getDouble( - quantileObj, + quantileObj, (PrimitiveObjectInspector)quantileOI); } for(int ii = 0; ii < result.length; ++ii) { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java (working copy) @@ -45,7 +45,7 @@ /** * Computes an approximate histogram of a numerical column using a user-specified number of bins. - * + * * The output is an array of (x,y) pairs as Hive struct objects that represents the histogram's * bin centers and heights. */ @@ -72,7 +72,7 @@ throw new UDFArgumentTypeException(parameters.length - 1, "Please specify exactly two arguments."); } - + // validate the first parameter, which is the expression to compute over if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, @@ -87,6 +87,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: break; case STRING: case BOOLEAN: @@ -170,7 +171,7 @@ @Override public Object terminatePartial(AggregationBuffer agg) throws HiveException { - // Return a single ArrayList where the first element is the number of histogram bins, + // Return a single ArrayList where the first element is the number of histogram bins, // and subsequent elements represent histogram (x,y) pairs. StdAgg myagg = (StdAgg) agg; return myagg.histogram.serialize(); @@ -233,7 +234,7 @@ } - // Aggregation buffer definition and manipulation methods + // Aggregation buffer definition and manipulation methods static class StdAgg implements AggregationBuffer { NumericHistogram histogram; // the histogram object }; Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java (working copy) @@ -102,6 +102,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -110,6 +111,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCorrelationEvaluator(); case STRING: case BOOLEAN: Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java (working copy) @@ -93,6 +93,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -101,6 +102,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceEvaluator(); case STRING: case BOOLEAN: Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java (working copy) @@ -28,7 +28,7 @@ /** * Compute the sample standard deviation by extending GenericUDAFVariance and * overriding the terminate() method of the evaluator. - * + * */ @Description(name = "stddev_samp", value = "_FUNC_(x) - Returns the sample standard deviation of a set of numbers") @@ -55,6 +55,7 @@ case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFStdSampleEvaluator(); case BOOLEAN: default: Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java (working copy) @@ -17,12 +17,15 @@ */ package org.apache.hadoop.hive.ql.udf.generic; +import java.math.BigDecimal; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -66,6 +69,8 @@ case DOUBLE: case STRING: return new GenericUDAFSumDouble(); + case DECIMAL: + return new GenericUDAFSumBigDecimal(); case BOOLEAN: default: throw new UDFArgumentTypeException(0, @@ -75,6 +80,89 @@ } /** + * GenericUDAFSumBigDecimal. + * + */ + public static class GenericUDAFSumBigDecimal extends GenericUDAFEvaluator { + private PrimitiveObjectInspector inputOI; + private BigDecimalWritable result; + + @Override + public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { + assert (parameters.length == 1); + super.init(m, parameters); + result = new BigDecimalWritable(BigDecimal.ZERO); + inputOI = (PrimitiveObjectInspector) parameters[0]; + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + /** class for storing double sum value. */ + static class SumBigDecimalAgg implements AggregationBuffer { + boolean empty; + BigDecimal sum; + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + SumBigDecimalAgg agg = new SumBigDecimalAgg(); + reset(agg); + return agg; + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg bdAgg = (SumBigDecimalAgg) agg; + bdAgg.empty = true; + bdAgg.sum = BigDecimal.ZERO; + } + + boolean warned = false; + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { + assert (parameters.length == 1); + try { + merge(agg, parameters[0]); + } catch (NumberFormatException e) { + if (!warned) { + warned = true; + LOG.warn(getClass().getSimpleName() + " " + + StringUtils.stringifyException(e)); + LOG + .warn(getClass().getSimpleName() + + " ignoring similar exceptions."); + } + } + } + + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException { + return terminate(agg); + } + + @Override + public void merge(AggregationBuffer agg, Object partial) throws HiveException { + if (partial != null) { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + myagg.empty = false; + myagg.sum = myagg.sum.add( + PrimitiveObjectInspectorUtils.getBigDecimal(partial, inputOI)); + } + } + + @Override + public Object terminate(AggregationBuffer agg) throws HiveException { + SumBigDecimalAgg myagg = (SumBigDecimalAgg) agg; + if (myagg.empty) { + return null; + } + result.set(myagg.sum); + return result; + } + + } + + /** * GenericUDAFSumDouble. * */ Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java (working copy) @@ -28,7 +28,7 @@ /** * Compute the sample variance by extending GenericUDAFVariance and overriding * the terminate() method of the evaluator. - * + * */ @Description(name = "var_samp", value = "_FUNC_(x) - Returns the sample variance of a set of numbers") @@ -56,6 +56,7 @@ case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceSampleEvaluator(); case BOOLEAN: default: Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java (working copy) @@ -43,7 +43,7 @@ /** * Compute the variance. This class is extended by: GenericUDAFVarianceSample * GenericUDAFStd GenericUDAFStdSample - * + * */ @Description(name = "variance,var_pop", value = "_FUNC_(x) - Returns the variance of a set of numbers") @@ -72,6 +72,7 @@ case DOUBLE: case STRING: case TIMESTAMP: + case DECIMAL: return new GenericUDAFVarianceEvaluator(); case BOOLEAN: default: @@ -85,18 +86,18 @@ * Evaluate the variance using the algorithm described by Chan, Golub, and LeVeque in * "Algorithms for computing the sample variance: analysis and recommendations" * The American Statistician, 37 (1983) pp. 242--247. - * + * * variance = variance1 + variance2 + n/(m*(m+n)) * pow(((m/n)*t1 - t2),2) - * + * * where: - variance is sum[x-avg^2] (this is actually n times the variance) * and is updated at every step. - n is the count of elements in chunk1 - m is - * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = + * the count of elements in chunk2 - t1 = sum of elements in chunk1, t2 = * sum of elements in chunk2. * * This algorithm was proven to be numerically stable by J.L. Barlow in * "Error analysis of a pairwise summation algorithm to compute sample variance" * Numer. Math, 58 (1991) pp. 583--590 - * + * */ public static class GenericUDAFVarianceEvaluator extends GenericUDAFEvaluator { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java (working copy) @@ -67,6 +67,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: switch (((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory()) { case BYTE: case SHORT: @@ -75,6 +76,7 @@ case FLOAT: case DOUBLE: case TIMESTAMP: + case DECIMAL: return new GenericUDAFCovarianceSampleEvaluator(); case STRING: case BOOLEAN: Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDecimal.java (working copy) @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.BigDecimalConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +@Description(name = "decimal", value = "_FUNC_(a) - cast a to decimal") +public class GenericUDFToDecimal extends GenericUDF { + + private PrimitiveObjectInspector argumentOI; + private BigDecimalConverter bdConverter; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length < 1) { + throw new UDFArgumentLengthException( + "The function DECIMAL requires at least one argument, got " + + arguments.length); + } + try { + argumentOI = (PrimitiveObjectInspector) arguments[0]; + } catch (ClassCastException e) { + throw new UDFArgumentException( + "The function DECIMAL takes only primitive types"); + } + + bdConverter = new BigDecimalConverter(argumentOI, + PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector); + return PrimitiveObjectInspectorFactory.writableBigDecimalObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + if (o0 == null) { + return null; + } + + return bdConverter.convert(o0); + } + + @Override + public String getDisplayString(String[] children) { + assert (children.length == 1); + StringBuilder sb = new StringBuilder(); + sb.append("CAST( "); + sb.append(children[0]); + sb.append(" AS DECIMAL)"); + return sb.toString(); + } + +} Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -35,7 +36,7 @@ * */ public class UDFToFloat extends UDF { - private FloatWritable floatWritable = new FloatWritable(); + private final FloatWritable floatWritable = new FloatWritable(); public UDFToFloat() { } @@ -184,4 +185,13 @@ } } + public FloatWritable evaluate(BigDecimalWritable i) { + if (i == null) { + return null; + } else { + floatWritable.set(i.getBigDecimal().floatValue()); + return floatWritable; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java (working copy) @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** @@ -33,7 +34,7 @@ public class UDFLog10 extends UDF { private static double log10 = Math.log(10.0); - private DoubleWritable result = new DoubleWritable(); + private final DoubleWritable result = new DoubleWritable(); public UDFLog10() { } @@ -50,4 +51,20 @@ } } + /** + * Returns the logarithm of "a" with base 10. + */ + public DoubleWritable evaluate(BigDecimalWritable a) { + if (a == null) { + return null; + } else { + double v = a.getBigDecimal().doubleValue(); + if (v < 0) { + return null; + } + result.set(Math.log(v) / log10); + return result; + } + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java (revision 1409060) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPlus.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf; import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.serde2.io.BigDecimalWritable; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; @@ -113,4 +114,14 @@ return doubleWritable; } + @Override + public BigDecimalWritable evaluate(BigDecimalWritable a, BigDecimalWritable b) { + if ((a == null) || (b == null)) { + return null; + } + + bigDecimalWritable.set(a.getBigDecimal().add(b.getBigDecimal())); + return bigDecimalWritable; + } + }